feat(keyboard): improve typing, voice flow, and polish reliability

Reduce extension memory pressure and delivery races while adding richer candidates, tactile feedback, and safer two-level creative polishing.
This commit is contained in:
Rocky
2026-08-05 21:39:31 +08:00
parent 38e5ad570d
commit 31f5937a7f
177 changed files with 8343 additions and 3904 deletions
@@ -0,0 +1,161 @@
// SevenDayUsageChart.swift
// OSGKeyboard · Shared
//
// 7-day dictation bar chart. Platform shells wrap this in their own page
// layout; the chart itself only needs points + UI language.
import Charts
import SwiftUI
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
public struct SevenDayUsageChart: View {
@Environment(\.themePalette) private var palette
public let points: [UsageStatisticsStore.DailyUsagePoint]
public let language: AppUILanguage
/// Bar area height. Phone stacked layout uses a shorter value.
public var chartMinHeight: CGFloat
/// When true, wrap content in `UsageSurfaceCard` (default). Pass false if
/// the caller already provides a surface.
public var embedsInCard: Bool
/// When true (iPad split), the chart fills all available height. When false
/// (phone stacked home), the bar area is a fixed `chartMinHeight` so the card
/// stays compact and does not steal space from surrounding content.
public var expands: Bool
public init(
points: [UsageStatisticsStore.DailyUsagePoint],
language: AppUILanguage,
chartMinHeight: CGFloat = 96,
embedsInCard: Bool = true,
expands: Bool = true
) {
self.points = points
self.language = language
self.chartMinHeight = chartMinHeight
self.embedsInCard = embedsInCard
self.expands = expands
}
private var total: Int {
points.reduce(0) { $0 + $1.value }
}
private var maxValue: Int {
max(points.map(\.value).max() ?? 0, 1)
}
private var chartLocale: Locale {
Locale(identifier: language.resolvedLanguageCode())
}
/// The real day for each bar; used to decide which axis marks get a label.
private var pointDates: Set<Date> {
Set(points.map(\.date))
}
/// Axis tick positions: the 7 real days plus one trailing boundary (last + 1
/// day) so `centered: true` has a span to center the final day's label within.
private var axisDates: [Date] {
let dates = points.map(\.date)
guard let last = dates.last,
let boundary = Calendar.current.date(byAdding: .day, value: 1, to: last)
else { return dates }
return dates + [boundary]
}
public var body: some View {
Group {
if embedsInCard {
UsageSurfaceCard(padding: Spacing.md) {
chartContent
}
} else {
chartContent
}
}
}
private var chartContent: some View {
VStack(alignment: .leading, spacing: Spacing.sm) {
header
chart
.frame(maxWidth: .infinity, maxHeight: expands ? .infinity : nil)
}
.frame(
maxWidth: .infinity,
maxHeight: expands ? .infinity : nil,
alignment: .topLeading
)
}
// MARK: - Header
private var header: some View {
HStack(alignment: .firstTextBaseline) {
VStack(alignment: .leading, spacing: 2) {
Text(SharedL10n.string("stat.weekChart.title", language: language).uppercased())
.font(TypeStyle.caption2)
.tracking(0.6)
.foregroundStyle(palette.textTertiary)
Text(SharedL10n.string("stat.weekChart.caption", language: language))
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
}
Spacer(minLength: Spacing.sm)
Text(UsageStatisticsStore.formatCount(total, language: language))
.font(TypeStyle.title2)
.foregroundStyle(palette.accent)
.lineLimit(1)
.minimumScaleFactor(0.7)
.contentTransition(.numericText())
.animation(Motion.soft, value: total)
}
}
// MARK: - Bars
@ViewBuilder
private var chart: some View {
if total == 0 {
Text(SharedL10n.string("stat.weekChart.empty", language: language))
.font(TypeStyle.caption)
.foregroundStyle(palette.textTertiary)
.frame(maxWidth: .infinity, maxHeight: expands ? .infinity : nil, alignment: .center)
.multilineTextAlignment(.center)
.frame(height: expands ? nil : chartMinHeight)
.frame(minHeight: expands ? chartMinHeight : nil)
} else {
Chart(points) { point in
BarMark(
x: .value("day", point.date, unit: .day),
y: .value("chars", point.value)
)
.cornerRadius(4)
.foregroundStyle(palette.accent.gradient)
}
.chartYScale(domain: 0...max(1, Int(ceil(Double(maxValue) * 1.15))))
.chartYAxis(.hidden)
.chartXAxis {
// Center each weekday label under its bar. Date `BarMark`s draw the
// bar centered within its day band, but axis labels default to the
// day's leading tick, so `centered: true` re-centers them on the bar.
//
// `centered` positions a label between its tick and the *next* tick,
// so the final day (Saturday) would be dropped for lack of a trailing
// tick. We append one boundary tick (last day + 1) to give it a span,
// and only draw labels for the real 7 days.
AxisMarks(values: axisDates) { value in
if let date = value.as(Date.self), pointDates.contains(date) {
AxisValueLabel(format: .dateTime.weekday(.narrow), centered: true)
}
}
}
.environment(\.locale, chartLocale)
.frame(height: expands ? nil : chartMinHeight)
.frame(minHeight: expands ? chartMinHeight : nil)
}
}
}
@@ -0,0 +1,146 @@
// SupportDeveloperSection.swift
// OSGKeyboard · Shared
//
// Optional voluntary tip block for Settings. Does not gate features.
import SwiftUI
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
/// iOS Settings card for the consumable support tip.
public struct SupportDeveloperSection: View {
@ObservedObject private var tipManager: TipPurchaseManager
@Environment(\.themePalette) private var palette
private let language: AppUILanguage
@State private var showThankYouAlert = false
@State private var showErrorAlert = false
@State private var errorMessage = ""
public init(
language: AppUILanguage,
tipManager: TipPurchaseManager = .shared
) {
self.language = language
self.tipManager = tipManager
}
public var body: some View {
CardSection(title: SharedL10n.string("tip.title", language: language)) {
VStack(alignment: .leading, spacing: Spacing.sm) {
SupportDeveloperTipBody(language: language)
if tipManager.supportCount > 0 {
Text(
SharedL10n.format(
"tip.thankYou.past",
language: language,
tipManager.supportCount
)
)
.font(TypeStyle.caption)
.foregroundStyle(palette.accent)
}
tipButton
.disabled(isPurchaseInFlight)
Text(SharedL10n.string("tip.consumableNotice", language: language))
.font(TypeStyle.caption2)
.foregroundStyle(palette.textTertiary)
.fixedSize(horizontal: false, vertical: true)
}
.padding(Spacing.md)
.frame(maxWidth: .infinity, alignment: .leading)
.surfaceCard()
}
.onChange(of: tipManager.purchaseState) { _, newValue in
switch newValue {
case .succeeded:
showThankYouAlert = true
case .failed(let message):
errorMessage = message
showErrorAlert = true
default:
break
}
}
.alert(
SharedL10n.string("tip.thankYou.title", language: language),
isPresented: $showThankYouAlert
) {
Button(SharedL10n.string("tip.alert.dismiss", language: language)) {
tipManager.acknowledgePurchaseState()
}
} message: {
Text(SharedL10n.string("tip.thankYou.message", language: language))
}
.alert(
SharedL10n.string("tip.error.title", language: language),
isPresented: $showErrorAlert
) {
Button(SharedL10n.string("tip.alert.dismiss", language: language)) {
tipManager.acknowledgePurchaseState()
}
} message: {
Text(errorMessage)
}
}
private var isPurchaseInFlight: Bool {
switch tipManager.purchaseState {
case .loading, .purchasing:
return true
default:
return false
}
}
private var tipButton: some View {
Button {
Task { await tipManager.purchase() }
} label: {
HStack(spacing: Spacing.sm) {
if isPurchaseInFlight {
ProgressView()
.controlSize(.small)
}
Text(tipButtonTitle)
.font(TypeStyle.body.weight(.semibold))
}
.frame(maxWidth: .infinity)
.padding(.vertical, Spacing.sm)
.foregroundStyle(.white)
.background(palette.accent, in: RoundedRectangle(cornerRadius: Radius.large, style: .continuous))
}
.buttonStyle(.plain)
.accessibilityLabel(tipButtonTitle)
}
private var tipButtonTitle: String {
if let product = tipManager.product {
return SharedL10n.format("tip.button", language: language, product.displayPrice)
}
return SharedL10n.string("tip.buttonFallback", language: language)
}
}
/// Shared copy + tip button for macOS Settings (Mac chrome wraps this).
public struct SupportDeveloperTipBody: View {
@Environment(\.themePalette) private var palette
private let language: AppUILanguage
public init(language: AppUILanguage) {
self.language = language
}
public var body: some View {
Text(SharedL10n.string("tip.body", language: language))
.font(TypeStyle.body)
.foregroundStyle(palette.textSecondary)
.fixedSize(horizontal: false, vertical: true)
}
}
@@ -0,0 +1,199 @@
// UsageStatsCluster.swift
// OSGKeyboard · Shared
//
// Cross-platform home / dashboard stats: 7-day chart + cumulative metrics.
// Callers observe their store and pass plain values Shared stays unbound
// from platform singletons.
import SwiftUI
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
public struct UsageStatsCluster: View {
@Environment(\.themePalette) private var palette
public enum Layout: Sendable, Equatable {
/// Chart left, 2×2 `UsageStatCard` grid right (Mac / iPad).
case split
/// Chart above a compact single-card 2×2 grid (iPhone).
case stacked
}
/// 2×2 沿 HomeStatsCard
static let compactGridHeight: CGFloat = 166
public let layout: Layout
public let language: AppUILanguage
public let points: [UsageStatisticsStore.DailyUsagePoint]
public let dictationCharacterCount: Int
public let dictationDurationSeconds: TimeInterval
public let translationCharacterCount: Int
public let dictionaryTermCount: Int
/// iPhone SE stacked
public let compact: Bool
public init(
layout: Layout,
language: AppUILanguage,
points: [UsageStatisticsStore.DailyUsagePoint],
dictationCharacterCount: Int,
dictationDurationSeconds: TimeInterval,
translationCharacterCount: Int,
dictionaryTermCount: Int,
compact: Bool = false
) {
self.layout = layout
self.language = language
self.points = points
self.dictationCharacterCount = dictationCharacterCount
self.dictationDurationSeconds = dictationDurationSeconds
self.translationCharacterCount = translationCharacterCount
self.dictionaryTermCount = dictionaryTermCount
self.compact = compact
}
public var body: some View {
switch layout {
case .split:
splitBody
case .stacked:
stackedBody
}
}
// MARK: - Split (Mac / iPad)
private var splitBody: some View {
HStack(alignment: .top, spacing: Spacing.md) {
SevenDayUsageChart(points: points, language: language)
.frame(maxWidth: .infinity, maxHeight: .infinity)
splitStatGrid
.frame(maxWidth: .infinity)
}
.fixedSize(horizontal: false, vertical: true)
}
private var splitStatGrid: some View {
VStack(spacing: Spacing.md) {
HStack(spacing: Spacing.md) {
UsageStatCard(
title: SharedL10n.string("stat.words", language: language),
value: UsageStatisticsStore.formatCount(dictationCharacterCount, language: language),
caption: SharedL10n.string("stat.transcribed", language: language),
systemImage: "text.alignleft",
accent: true
)
UsageStatCard(
title: SharedL10n.string("stat.dictationTime", language: language),
value: UsageStatisticsStore.formatDuration(dictationDurationSeconds, language: language),
caption: SharedL10n.string("stat.cumulativeDuration", language: language),
systemImage: "waveform"
)
}
HStack(spacing: Spacing.md) {
UsageStatCard(
title: SharedL10n.string("stat.translation", language: language),
value: UsageStatisticsStore.formatCount(translationCharacterCount, language: language),
caption: SharedL10n.string("stat.cumulativeTranslation", language: language),
systemImage: "character.bubble"
)
UsageStatCard(
title: SharedL10n.string("stat.dictionary", language: language),
value: UsageStatisticsStore.formatCount(dictionaryTermCount, language: language),
caption: SharedL10n.string("stat.customTerms", language: language),
systemImage: "character.book.closed"
)
}
}
}
// MARK: - Stacked (iPhone)
private var stackedBody: some View {
VStack(spacing: compact ? Spacing.sm : Spacing.md) {
SevenDayUsageChart(
points: points,
language: language,
chartMinHeight: compact ? 72 : 96,
expands: false
)
compactStatGrid
}
}
/// Phone-friendly 2×2: value + label only, single card with hairline dividers.
private var compactStatGrid: some View {
VStack(spacing: 0) {
HStack(spacing: 0) {
compactCell(
systemImage: "waveform",
value: UsageStatisticsStore.formatDuration(dictationDurationSeconds, language: language),
label: SharedL10n.string("stat.dictationTime", language: language)
)
compactDivider
compactCell(
systemImage: "text.alignleft",
value: UsageStatisticsStore.formatCount(dictationCharacterCount, language: language),
label: SharedL10n.string("stat.words", language: language)
)
}
Rectangle()
.fill(palette.divider)
.frame(height: 0.5)
HStack(spacing: 0) {
compactCell(
systemImage: "character.bubble",
value: UsageStatisticsStore.formatCount(translationCharacterCount, language: language),
label: SharedL10n.string("stat.translation", language: language)
)
compactDivider
compactCell(
systemImage: "character.book.closed",
value: UsageStatisticsStore.formatCount(dictionaryTermCount, language: language),
label: SharedL10n.string("stat.dictionary", language: language)
)
}
}
// HomeStatsCard 166pt
.frame(height: UsageStatsCluster.compactGridHeight)
.background(palette.surface)
.clipShape(RoundedRectangle(cornerRadius: Radius.xl, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: Radius.xl, style: .continuous)
.stroke(palette.divider, lineWidth: 0.5)
)
}
private var compactDivider: some View {
Rectangle()
.fill(palette.divider)
.frame(width: 0.5)
}
private func compactCell(systemImage: String, value: String, label: String) -> some View {
HStack(alignment: .top, spacing: Spacing.xs) {
VStack(alignment: .leading, spacing: Spacing.xs) {
Text(value)
.font(.system(size: 24, weight: .semibold, design: .rounded))
.foregroundStyle(palette.textPrimary)
.lineLimit(1)
.minimumScaleFactor(0.75)
.contentTransition(.numericText())
.animation(Motion.soft, value: value)
Text(label)
.font(TypeStyle.caption2)
.foregroundStyle(palette.textSecondary)
.lineLimit(1)
.minimumScaleFactor(0.85)
}
Spacer(minLength: Spacing.xs)
Image(systemName: systemImage)
.font(.system(size: 18, weight: .semibold))
.foregroundStyle(palette.accent)
.padding(.top, 2)
}
.frame(maxWidth: .infinity, alignment: .leading)
.padding(Spacing.md)
}
}
+22
View File
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>FMWK</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
</dict>
</plist>
@@ -0,0 +1,25 @@
// AudioBufferSnapshot+AVFoundation.swift
// OSGKeyboard · HostSupport
//
// AVAudioPCMBuffer AudioBufferSnapshot copy helper. Kept out of Shared so
// the keyboard extension does not link AVFoundation for this type alone.
import AVFoundation
import Foundation
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
extension AudioBufferSnapshot {
/// Construct from an `AVAudioPCMBuffer` by copying out the channel data.
public init(buffer: AVAudioPCMBuffer) {
guard let channelData = buffer.floatChannelData else {
self.init(samples: [], sampleRate: buffer.format.sampleRate)
return
}
let n = Int(buffer.frameLength)
var copy = [Float](repeating: 0, count: n)
memcpy(&copy, channelData[0], n * MemoryLayout<Float>.size)
self.init(samples: copy, sampleRate: buffer.format.sampleRate)
}
}
@@ -0,0 +1,28 @@
// ASRChunkTranscribing.swift
// OSGKeyboard · Shared
//
// Minimal ASR surface for pipelined utterance chunking. Keeps
// `ChunkedUtterancePipeline` independent of iOS-only `SpeechAnalyzer`.
import Foundation
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
public enum ASRChunkResult: Sendable, Equatable {
case success(String)
case failure(String)
case cancelled
}
/// One-shot chunk transcription used by `ChunkedUtterancePipeline`.
public protocol ASRChunkTranscribing: Sendable {
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult
func cancel()
func resetForNewUtterance()
}
extension ASRChunkTranscribing {
public func cancel() {}
public func resetForNewUtterance() {}
}
@@ -0,0 +1,677 @@
// ASRService.swift
// OSGKeyboard · Shared
//
// Speech-to-text abstraction. As of iOS 26 being the minimum
// deployment target, the only ASR backend is `SpeechAnalyzer` +
// `DictationTranscriber` always on-device, no cloud fallback, no
// `requiresOnDevice` toggle. The previous legacy recognizer path is
// gone; if a future platform ever needs it back,
// reintroduce as a sibling class in `ASRServiceFactory.make()`.
//
// Lives in `OSGKeyboardShared` (not the keyboard extension target) so
// that the host app's `KeyboardPreviewSheet` can run the same ASR
// pipeline against real iOS audio without it, the in-app preview
// was a static mock that never actually called `SFSpeechRecognizer`,
// and "did you actually wire up ASR?" was a fair review note.
import Foundation
import AVFoundation
import CoreMedia
import Speech
import os
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
// MARK: - Sendable conformance
// `AVAudioPCMBuffer` and `SpeechAnalyzer` are not Sendable. We only
// ever access them serially the PCM buffer is built and consumed
// inside a single Task, and the analyzer is cancelled but never
// shared concurrently so an unchecked conformance is sound here.
extension AVAudioPCMBuffer: @unchecked @retroactive Sendable {}
// MARK: - Protocol
public protocol ASRService: ASRChunkTranscribing, Sendable {
/// Start a transcription session. The returned stream emits `.partial`
/// updates and exactly one `.final` (or `.error`) before finishing.
/// `SpeechAnalyzer` is always fully on-device, so there is no
/// `requiresOnDevice` flag that legacy cloud-fallback control
/// doesn't apply to the iOS 26 `SpeechAnalyzer` path.
func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent>
/// Cancel any in-flight recognition and tear down its tasks.
func cancel()
/// Clears cancellation / cached session state before a new utterance.
func resetForNewUtterance()
/// Pre-load locale assets and analyzer format for lower first-chunk latency.
func warmup(locale: Locale) async
/// Transcribe one PCM chunk (Flow pipelined path). Default wraps `transcribe(stream:)`.
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult
}
extension ASRService {
public func resetForNewUtterance() {}
public func warmup(locale: Locale) async {}
public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
guard !samples.isEmpty else { return .success("") }
if Task.isCancelled { return .cancelled }
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
continuation.yield(snapshot)
continuation.finish()
var lastPartial = ""
var finalText = ""
var failure: String?
for await event in transcribe(stream: stream, locale: locale) {
if Task.isCancelled { return .cancelled }
switch event {
case .capability:
break
case .partial(let text):
lastPartial = text
case .final(let text):
finalText = text
case .error(let message):
failure = message
}
}
if let failure {
return .failure(failure)
}
let trimmed = finalText.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty {
return .success(trimmed)
}
let partial = lastPartial.trimmingCharacters(in: .whitespacesAndNewlines)
if !partial.isEmpty {
return .success(partial)
}
return .success("")
}
}
public enum ASREvent: Sendable, Equatable {
/// Emitted exactly once at the start of every `transcribe` call, so
/// the UI can flag non-on-device locales (e.g. ja-JP on devices that
/// only ship on-device ASR for en/zh). The ASR session continues
/// either way we fall back to cloud automatically.
case capability(onDeviceSupported: Bool)
case partial(String)
case final(String)
case error(String)
}
// MARK: - Factory
public enum ASRServiceFactory {
/// Returns on-device SpeechAnalyzer for `local`, or the user's cloud
/// ASR provider when `engineMode == "cloud"`.
public static func make(store: any ConfigurationStore = AppGroupStore()) -> ASRService {
if store.engineMode == "cloud" {
return CloudASRService(store: store)
}
return SpeechAnalyzerASR()
}
}
// MARK: - PCM format conversion (testable helpers)
//
// Extracted from the audio-thread hot path so the scaling + clipping
// math can be exercised in unit tests without instantiating the
// full ASR pipeline. See `OSGKeyboardTests/ASRConversionTests.swift`.
extension ASRServiceFactory {
/// Convert a Float32 PCM buffer (`-1.0...1.0`) to an Int16 PCM
/// buffer (`-32768...32767`).
///
/// - Parameters:
/// - source: Pointer to `sourceCount` `Float` samples. May be
/// `nil` when `sourceCount == 0`.
/// - sourceCount: Number of samples to convert. A `0` count
/// turns the call into a no-op regardless of the pointers.
/// - destination: Pointer to at least `sourceCount` slots of
/// `Int16`. May be `nil` when `sourceCount == 0`.
///
/// Per-sample: `Int16(round(clamp(s * 32767, -32768, 32767)))`.
/// The explicit clip matters: without it, `s == 1.0` would map
/// to `+32767` (fine) but `s == 1.5` (which can show up at the
/// audio engine boundary under gain) would wrap to a negative
/// value after the implicit FloatInt16 conversion. The
/// `round()` (rather than truncate) preserves DC balance `0.5`
/// quantises to `+16384`, not `+16383`, matching what most audio
/// DAW round-trips expect.
static func convertFloat32ToInt16(
source: UnsafePointer<Float>?,
sourceCount: Int,
destination: UnsafeMutablePointer<Int16>?
) {
guard sourceCount > 0, let source, let destination else { return }
for i in 0..<sourceCount {
let scaled = source[i] * 32767.0
let clipped = Swift.max(-32768.0, Swift.min(32767.0, scaled))
destination[i] = Int16(clipped.rounded())
}
}
}
// MARK: - SpeechAnalyzer implementation (iOS 26+)
/// ASR backend that uses the iOS 26 `SpeechAnalyzer` + `DictationTranscriber`
/// APIs. This engine is always fully on-device.
final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
private let lock = OSAllocatedUnfairLock()
private var analyzer: SpeechAnalyzer?
private var analyzerTask: Task<Void, Never>?
private var analyzerFinished = false
/// Reused across pipelined chunks within one utterance (assets + format).
private var chunkPreparedLocaleID: String?
private var chunkAnalyzerFormat: AVAudioFormat?
func resetForNewUtterance() {
// Keep chunk format / asset cache warm across utterances in one Flow session.
}
func invalidateChunkPreparationCache() {
lock.withLock {
chunkPreparedLocaleID = nil
chunkAnalyzerFormat = nil
}
}
func warmup(locale: Locale) async {
let warmupStartedAt = Date()
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
Self.debug("warmup locale unsupported requested=\(locale.identifier(.bcp47))")
FlowTrace.warn(
"asr.local.warmup.localeUnsupported",
"requested=\(locale.identifier(.bcp47))"
)
return
}
let localeID = resolvedLocale.identifier(.bcp47)
let cachedLocaleID = lock.withLock { chunkPreparedLocaleID }
if cachedLocaleID == localeID, lock.withLock({ chunkAnalyzerFormat != nil }) {
Self.debug("warmup cache hit locale=\(localeID)")
FlowTrace.asr("local.warmup.cacheHit", "locale=\(localeID)")
return
}
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
Self.debug(
"warmup start locale=\(localeID) customLMEnabled=\(setup.customLanguageModelEnabled) " +
"customLMAttached=\(setup.usesCustomLanguageModel) " +
"clmState=\(Self.describeCLMState(setup.clmState))"
)
FlowTrace.asr(
"local.warmup.begin",
"locale=\(localeID) customLM=\(setup.usesCustomLanguageModel ? 1 : 0) "
+ "clmState=\(Self.describeCLMState(setup.clmState))"
)
do {
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
compatibleWith: [setup.transcriber],
considering: Self.captureFormat
) else {
Self.debug("warmup format unsupported locale=\(localeID)")
FlowTrace.warn("asr.local.warmup.formatUnsupported", "locale=\(localeID)")
return
}
lock.withLock {
chunkPreparedLocaleID = localeID
chunkAnalyzerFormat = format
}
Self.debug("warmup ready locale=\(localeID)")
FlowTrace.asr(
"local.warmup.ready",
"locale=\(localeID) analyzerRate=\(Int(format.sampleRate)) "
+ "elapsed=\(FlowTrace.seconds(since: warmupStartedAt))s"
)
} catch {
Self.debug("warmup failed: \(error.localizedDescription)")
FlowTrace.warn(
"asr.local.warmup.failed",
"locale=\(localeID) elapsed=\(FlowTrace.seconds(since: warmupStartedAt))s "
+ "error=\(error.localizedDescription)"
)
}
}
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
guard !samples.isEmpty else { return .success("") }
if Task.isCancelled { return .cancelled }
let startedAt = Date()
let rms = Self.rms(of: samples)
Self.debug(
"chunk start samples=\(samples.count) rms=\(String(format: "%.4f", rms)) " +
"locale=\(locale.identifier(.bcp47))"
)
do {
let text = try await transcribeSamples(samples, locale: locale, reuseChunkPrep: true)
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
Self.debug(
"chunk success textLen=\(trimmed.count) elapsed=\(Self.elapsed(startedAt))s " +
"empty=\(trimmed.isEmpty)"
)
FlowTrace.transcript(
"asr.local.chunk",
trimmed,
"engine=local samples=\(samples.count) rms=\(String(format: "%.4f", rms)) "
+ "elapsed=\(Self.elapsed(startedAt))s locale=\(locale.identifier(.bcp47))"
)
return trimmed.isEmpty ? .success("") : .success(trimmed)
} catch is CancellationError {
Self.debug("chunk cancelled elapsed=\(Self.elapsed(startedAt))s")
FlowTrace.asr("local.chunk.cancelled", "samples=\(samples.count)")
return .cancelled
} catch {
Self.debug("chunk failed elapsed=\(Self.elapsed(startedAt))s error=\(error.localizedDescription)")
FlowTrace.warn(
"asr.local.chunk.failed",
"samples=\(samples.count) rms=\(String(format: "%.4f", rms)) "
+ "error=\(error.localizedDescription)"
)
return .failure(error.localizedDescription)
}
}
/// Analyze a single PCM buffer without the streaming `transcribe` wrapper.
private func transcribeSamples(
_ samples: [Float],
locale: Locale,
reuseChunkPrep: Bool
) async throws -> String {
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
throw ASRChunkError.localeUnsupported
}
let localeID = resolvedLocale.identifier(.bcp47)
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
Self.debug(
"chunk setup locale=\(localeID) customLMEnabled=\(setup.customLanguageModelEnabled) " +
"customLMAttached=\(setup.usesCustomLanguageModel) " +
"clmState=\(Self.describeCLMState(setup.clmState))"
)
let analyzerFormat: AVAudioFormat
let cachedPrep = lock.withLock { (chunkPreparedLocaleID, chunkAnalyzerFormat) }
if reuseChunkPrep,
cachedPrep.0 == localeID,
let cached = cachedPrep.1 {
analyzerFormat = cached
Self.debug(
"chunk using cached analyzer format sr=\(Int(cached.sampleRate)) " +
"channels=\(cached.channelCount) common=\(cached.commonFormat.rawValue)"
)
} else {
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
compatibleWith: [setup.transcriber],
considering: Self.captureFormat
) else {
throw ASRChunkError.formatUnsupported
}
analyzerFormat = format
lock.withLock {
chunkPreparedLocaleID = localeID
chunkAnalyzerFormat = format
}
Self.debug(
"chunk prepared analyzer format sr=\(Int(format.sampleRate)) " +
"channels=\(format.channelCount) common=\(format.commonFormat.rawValue)"
)
}
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
guard let pcm = Self.makeAnalyzerPCMBuffer(from: snapshot, format: analyzerFormat) else {
throw ASRChunkError.formatUnsupported
}
let analyzer = SpeechAnalyzer(modules: [setup.transcriber])
try await analyzer.prepareToAnalyze(in: analyzerFormat)
let resultsTask = Task<String, Error> {
var accumulator = ProgressiveDictationTranscriptAccumulator()
for try await result in setup.transcriber.results {
if Task.isCancelled { break }
let text = String(result.text.characters)
_ = accumulator.ingest(range: result.range, text: text)
}
return accumulator.finalize()
}
let inputStream = AsyncStream<AnalyzerInput> { continuation in
continuation.yield(AnalyzerInput(buffer: pcm))
continuation.finish()
}
let lastSampleTime = try await analyzer.analyzeSequence(inputStream)
if let lastSampleTime {
try await analyzer.finalizeAndFinish(through: lastSampleTime)
} else {
await analyzer.cancelAndFinishNow()
}
return try await resultsTask.value
}
private enum ASRChunkError: LocalizedError {
case localeUnsupported
case formatUnsupported
var errorDescription: String? {
switch self {
case .localeUnsupported:
return SharedL10n.string("error.asr.localeUnsupported")
case .formatUnsupported:
return SharedL10n.string("error.asr.formatUnsupported")
}
}
}
/// Canonical capture format: 16 kHz mono Float32 from `AudioCaptureService`
/// / `PreviewASRController` before it reaches SpeechAnalyzer.
private static let captureFormat = AVAudioFormat(
commonFormat: .pcmFormatFloat32,
sampleRate: 16_000,
channels: 1,
interleaved: false
)!
func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent> {
AsyncStream { continuation in
continuation.yield(.capability(onDeviceSupported: true))
let task = Task { [weak self] in
guard let self else { return }
self.lock.withLock { self.analyzerFinished = false }
defer {
self.lock.withLock {
self.analyzer = nil
self.analyzerTask = nil
self.analyzerFinished = true
}
}
do {
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
Self.debug("locale unsupported: \(locale.identifier(.bcp47))")
continuation.yield(.error(SharedL10n.string("error.asr.localeUnsupported")))
continuation.finish()
return
}
// Each pipelined chunk is 30 s; long dictation preset keeps a
// single chunk coherent (Flow utterances run up to 3 min).
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
Self.debug(
"stream setup locale=\(resolvedLocale.identifier(.bcp47)) " +
"customLMEnabled=\(setup.customLanguageModelEnabled) " +
"customLMAttached=\(setup.usesCustomLanguageModel) " +
"clmState=\(Self.describeCLMState(setup.clmState))"
)
do {
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
} catch {
Self.debug("asset prepare failed: \(error.localizedDescription)")
FlowTrace.warn(
"asr.local.stream.assetsNotReady",
"locale=\(resolvedLocale.identifier(.bcp47)) "
+ "error=\(error.localizedDescription)"
)
continuation.yield(.error(SharedL10n.string("error.asr.assetsNotReady")))
continuation.finish()
return
}
let newAnalyzer = SpeechAnalyzer(modules: [setup.transcriber])
self.lock.withLock { self.analyzer = newAnalyzer }
guard let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(
compatibleWith: [setup.transcriber],
considering: Self.captureFormat
) else {
continuation.yield(.error(SharedL10n.string("error.asr.formatUnsupported")))
continuation.finish()
return
}
try await newAnalyzer.prepareToAnalyze(in: analyzerFormat)
let inputStream = self.makeInputStream(from: stream, analyzerFormat: analyzerFormat)
// Apple recommends consuming `transcriber.results` concurrently
// while `analyzeSequence` drains the input stream.
let resultsTask = Task<String, Error> {
var accumulator = ProgressiveDictationTranscriptAccumulator()
for try await result in setup.transcriber.results {
if Task.isCancelled { break }
let text = String(result.text.characters)
guard let full = accumulator.ingest(range: result.range, text: text) else {
continue
}
FlowTrace.transcript("asr.local.partial", full, "engine=local")
continuation.yield(.partial(full))
}
return accumulator.finalize()
}
let lastSampleTime = try await newAnalyzer.analyzeSequence(inputStream)
if let lastSampleTime {
try await newAnalyzer.finalizeAndFinish(through: lastSampleTime)
} else {
await newAnalyzer.cancelAndFinishNow()
}
let lastText: String
do {
lastText = try await resultsTask.value
} catch {
Self.debug("transcriber results failed: \(error.localizedDescription)")
continuation.yield(.error(error.localizedDescription))
continuation.finish()
return
}
let trimmed = lastText.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty {
FlowTrace.warn(
"asr.local.stream.emptyFinal",
"locale=\(resolvedLocale.identifier(.bcp47))"
)
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
} else {
FlowTrace.transcript("asr.local.final", trimmed, "engine=local")
continuation.yield(.final(trimmed))
}
continuation.finish()
} catch is CancellationError {
continuation.finish()
} catch {
Self.debug("SpeechAnalyzer failed: \(error.localizedDescription)")
continuation.yield(.error(error.localizedDescription))
continuation.finish()
}
}
self.lock.withLock { self.analyzerTask = task }
continuation.onTermination = { @Sendable [weak self] _ in
self?.cancel()
}
}
}
private struct DiagnosticTranscriber {
let transcriber: DictationTranscriber
let customLanguageModelEnabled: Bool
let usesCustomLanguageModel: Bool
let clmState: CustomLanguageModelManager.PrepareState
}
private static func makeDiagnosticTranscriber(locale: Locale) -> DiagnosticTranscriber {
let defaults = AppGroup.defaultsIfAvailable
let clmKey = AppGroupConfiguration.Keys.localASRCustomLanguageModelEnabled
let clmEnabled = defaults?.object(forKey: clmKey) == nil
? true
: (defaults?.bool(forKey: clmKey) ?? true)
let clmState = CustomLanguageModelManager.shared.currentState()
let lmConfiguration = clmEnabled
? CustomLanguageModelManager.shared.configurationForTranscription(locale: locale)
: nil
let transcriber = CustomLanguageModelManager.makeDictationTranscriber(
locale: locale,
lmConfiguration: lmConfiguration
)
return DiagnosticTranscriber(
transcriber: transcriber,
customLanguageModelEnabled: clmEnabled,
usesCustomLanguageModel: lmConfiguration != nil,
clmState: clmState
)
}
private static func describeCLMState(_ state: CustomLanguageModelManager.PrepareState) -> String {
switch state {
case .idle:
return "idle"
case .preparing:
return "preparing"
case .ready:
return "ready"
case .failed(let message):
return "failed(\(message))"
}
}
private static func rms(of samples: [Float]) -> Float {
guard !samples.isEmpty else { return 0 }
var sum: Float = 0
for sample in samples {
sum += sample * sample
}
return sqrtf(sum / Float(samples.count))
}
private static func elapsed(_ start: Date) -> String {
String(format: "%.2f", Date().timeIntervalSince(start))
}
private static func debug(_ message: String) {
OSGLog.asr.info("\(message, privacy: .public)")
}
private static func prepareAssetsIfNeeded(
for transcriber: DictationTranscriber,
locale: Locale
) async throws {
let localeID = locale.identifier(.bcp47)
let startedAt = Date()
do {
_ = try await AssetInventory.reserve(locale: locale)
Self.debug("asset reserve ok locale=\(localeID)")
} catch {
// Reservation may already exist or slots are full; continue, but
// log it so local-ASR setup failures are not hidden behind a later
// "no speech" timeout.
Self.debug("asset reserve non-fatal locale=\(localeID) error=\(error.localizedDescription)")
}
do {
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
Self.debug("asset install required locale=\(localeID)")
try await request.downloadAndInstall()
Self.debug("asset install done locale=\(localeID) elapsed=\(elapsed(startedAt))s")
} else {
Self.debug("asset already installed locale=\(localeID) elapsed=\(elapsed(startedAt))s")
}
} catch {
Self.debug("asset prepare failed locale=\(localeID) elapsed=\(elapsed(startedAt))s error=\(error.localizedDescription)")
throw error
}
}
func cancel() {
let (task, currentAnalyzer, finished) = lock.withLock { () -> (Task<Void, Never>?, SpeechAnalyzer?, Bool) in
let t = analyzerTask
let a = analyzer
let f = analyzerFinished
analyzerTask = nil
analyzer = nil
return (t, a, f)
}
task?.cancel()
guard !finished, let currentAnalyzer else { return }
Task { await currentAnalyzer.cancelAndFinishNow() }
}
/// Maps 16 kHz Float32 snapshots into `AnalyzerInput` using the format
/// returned by `bestAvailableAudioFormat(compatibleWith:considering:)`.
private func makeInputStream(
from stream: AsyncStream<AudioBufferSnapshot>,
analyzerFormat: AVAudioFormat
) -> AsyncStream<AnalyzerInput> {
AsyncStream { continuation in
Task {
for await snap in stream {
guard !snap.samples.isEmpty else { continue }
guard let pcm = Self.makeAnalyzerPCMBuffer(from: snap, format: analyzerFormat) else {
continue
}
continuation.yield(AnalyzerInput(buffer: pcm))
}
continuation.finish()
}
}
}
private static func makeAnalyzerPCMBuffer(
from snap: AudioBufferSnapshot,
format: AVAudioFormat
) -> AVAudioPCMBuffer? {
let capacity = AVAudioFrameCount(snap.samples.count)
guard capacity > 0,
let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else {
return nil
}
pcm.frameLength = capacity
switch format.commonFormat {
case .pcmFormatInt16:
guard let dst = pcm.int16ChannelData?[0] else { return nil }
snap.samples.withUnsafeBufferPointer { src in
ASRServiceFactory.convertFloat32ToInt16(
source: src.baseAddress,
sourceCount: src.count,
destination: dst
)
}
case .pcmFormatFloat32:
guard let dst = pcm.floatChannelData?[0] else { return nil }
snap.samples.withUnsafeBufferPointer { src in
guard let base = src.baseAddress else { return }
memcpy(dst, base, src.count * MemoryLayout<Float>.stride)
}
default:
return nil
}
return pcm
}
}
@@ -0,0 +1,380 @@
// ChunkedUtterancePipeline.swift
// OSGKeyboard · Shared
//
// Pipelined Flow utterance ASR: split PCM while recording, transcribe chunks
// serially on a background queue, stitch partials for display and delivery.
import Foundation
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
public struct ChunkedUtteranceSuccess: Sendable, Equatable {
public let text: String
/// Same transcript with internal pause markers, used only by polish.
public let textWithPauseMarks: String
/// Non-fatal per-chunk ASR issues (delivered as soft warning when non-empty).
public let chunkWarnings: [String]
public init(
text: String,
textWithPauseMarks: String? = nil,
chunkWarnings: [String] = []
) {
self.text = text
self.textWithPauseMarks = textWithPauseMarks ?? text
self.chunkWarnings = chunkWarnings
}
}
public enum ChunkedUtterancePipelineOutcome: Sendable, Equatable {
case success(ChunkedUtteranceSuccess)
case failure(String)
case cancelled
}
/// Thread-safe queue between the chunk feeder and ASR worker.
private actor ChunkWorkQueue {
private var items: [UtteranceAudioChunk] = []
private var finished = false
private var waiters: [CheckedContinuation<UtteranceAudioChunk?, Never>] = []
func enqueue(_ chunk: UtteranceAudioChunk) {
items.append(chunk)
resumeWaiters()
}
func markFinished() {
finished = true
resumeWaiters()
}
func dequeue() async -> UtteranceAudioChunk? {
if !items.isEmpty {
return items.removeFirst()
}
if finished {
return nil
}
return await withCheckedContinuation { continuation in
waiters.append(continuation)
}
}
private func resumeWaiters() {
while !waiters.isEmpty {
if !items.isEmpty {
let waiter = waiters.removeFirst()
waiter.resume(returning: items.removeFirst())
} else if finished {
let waiter = waiters.removeFirst()
waiter.resume(returning: nil)
} else {
break
}
}
}
}
public actor ChunkedUtterancePipeline {
private let asr: any ASRChunkTranscribing
private let locale: Locale
private let config: FlowUtteranceChunkConfig
private var cancelled = false
public init(
asr: any ASRChunkTranscribing,
locale: Locale,
config: FlowUtteranceChunkConfig = .flowDefault
) {
self.asr = asr
self.locale = locale
self.config = config
}
public func cancel() {
cancelled = true
asr.cancel()
}
/// Consume `stream` until finished; ASR runs off the caller's actor while recording continues.
public func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
onPartial: @Sendable @escaping (String) -> Void
) async -> ChunkedUtterancePipelineOutcome {
asr.resetForNewUtterance()
let queue = ChunkWorkQueue()
var stitcher = UtteranceTranscriptStitcher()
var chunkWarnings: [String] = []
var failedChunks = 0
var processedChunks = 0
var previousChunkSamples: [Float] = []
var lastChunkSamples = 0
var didRetryEmptyFinal = false
let feeder = Task {
for await chunk in UtteranceStreamChunker.chunks(from: stream, config: config) {
if Task.isCancelled { break }
await queue.enqueue(chunk)
}
await queue.markFinished()
}
while true {
if cancelled || Task.isCancelled {
feeder.cancel()
return .cancelled
}
guard let chunk = await queue.dequeue() else { break }
if chunk.isLast && chunk.samples.isEmpty {
continue
}
processedChunks += 1
lastChunkSamples = chunk.samples.count
if let preMerge = FinalChunkRecovery.preMergePlan(
chunk: chunk,
processedChunks: processedChunks,
previousChunkSamples: previousChunkSamples,
config: config
) {
FlowPipelineDiagnostics.logFinalChunkRecovery(
action: "preMerge",
chunkIndex: chunk.index
)
let mergedResult = await transcribeChunkWithRetry(
samples: preMerge.samples,
chunkIndex: chunk.index
)
switch mergedResult {
case .success(let text):
// Empty / whitespace merge must NOT wipe a prior good segment
// (`append` ignores empty text, so remove-then-append would
// silently drop the only transcript the AC327-style bug).
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty {
FlowPipelineDiagnostics.logFinalChunkRecovery(
action: "preMergeKeepPrior",
chunkIndex: chunk.index
)
} else {
stitcher.removeLastSegment()
stitcher.append(
index: preMerge.stitchIndex,
text: text,
trailingPauseSeconds: chunk.trailingPauseSeconds
)
publishPartial(from: stitcher, onPartial: onPartial)
}
case .failure(let message):
// Keep prior stitcher text; treat as a soft chunk warning.
failedChunks += 1
chunkWarnings.append(
SharedL10n.format(
"error.asr.chunkFailed",
chunk.index + 1,
message
)
)
case .cancelled:
feeder.cancel()
return .cancelled
}
previousChunkSamples = chunk.samples
continue
}
let result = await transcribeChunkWithRetry(
samples: chunk.samples,
chunkIndex: chunk.index
)
logChunkOutcome(chunk: chunk, result: result)
switch result {
case .success(let text):
if chunk.isLast,
!didRetryEmptyFinal,
let retry = FinalChunkRecovery.emptyResultRetryPlan(
chunk: chunk,
previousChunkSamples: previousChunkSamples,
config: config,
asrText: text
) {
didRetryEmptyFinal = true
FlowPipelineDiagnostics.logFinalChunkRecovery(
action: "emptyRetry",
chunkIndex: chunk.index
)
let retryResult = await transcribeChunkWithRetry(
samples: retry.samples,
chunkIndex: chunk.index
)
switch retryResult {
case .success(let retryText):
let trimmed = retryText.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty {
if retry.stitchIndex < chunk.index {
stitcher.removeLastSegment()
}
stitcher.append(
index: retry.stitchIndex,
text: retryText,
trailingPauseSeconds: chunk.trailingPauseSeconds
)
publishPartial(from: stitcher, onPartial: onPartial)
} else {
stitcher.append(
index: chunk.index,
text: text,
trailingPauseSeconds: chunk.trailingPauseSeconds
)
publishPartial(from: stitcher, onPartial: onPartial)
}
case .failure(let message):
stitcher.append(
index: chunk.index,
text: text,
trailingPauseSeconds: chunk.trailingPauseSeconds
)
publishPartial(from: stitcher, onPartial: onPartial)
failedChunks += 1
chunkWarnings.append(
SharedL10n.format(
"error.asr.chunkFailed",
chunk.index + 1,
message
)
)
case .cancelled:
feeder.cancel()
return .cancelled
}
} else {
stitcher.append(
index: chunk.index,
text: text,
trailingPauseSeconds: chunk.trailingPauseSeconds
)
publishPartial(from: stitcher, onPartial: onPartial)
}
case .failure(let message):
failedChunks += 1
chunkWarnings.append(
SharedL10n.format(
"error.asr.chunkFailed",
chunk.index + 1,
message
)
)
case .cancelled:
feeder.cancel()
return .cancelled
}
previousChunkSamples = chunk.samples
}
_ = await feeder.value
let finalText = stitcher.composedSafely().trimmingCharacters(in: .whitespacesAndNewlines)
let markedText = stitcher.composedWithPauseMarks()
.trimmingCharacters(in: .whitespacesAndNewlines)
FlowPipelineDiagnostics.logChunkFinalize(
chunkCount: processedChunks,
lastChunkSamples: lastChunkSamples,
stitchedLength: finalText.count,
chunkWarnings: chunkWarnings.count
)
if finalText.isEmpty {
FlowTrace.warn(
"pipeline.stitch.empty",
"chunks=\(processedChunks) failedChunks=\(failedChunks) "
+ "lastChunkSamples=\(lastChunkSamples) warnings=\(chunkWarnings.count)"
)
if failedChunks > 0, processedChunks == failedChunks {
return .failure(SharedL10n.string("error.asr.noSpeech"))
}
return .failure(SharedL10n.string("error.asr.noSpeech"))
}
FlowTrace.transcript(
"asr.stitched",
finalText,
"chunks=\(processedChunks) failedChunks=\(failedChunks) warnings=\(chunkWarnings.count)"
)
return .success(
ChunkedUtteranceSuccess(
text: finalText,
textWithPauseMarks: markedText,
chunkWarnings: chunkWarnings
)
)
}
private func transcribeChunk(samples: [Float]) async -> ASRChunkResult {
let asr = self.asr
let locale = self.locale
return await Task.detached(priority: .userInitiated) {
await asr.transcribeChunk(samples: samples, locale: locale)
}.value
}
/// Retry one failed chunk before advancing the serial worker. Keeping the
/// same PCM samples prevents a transient request failure from creating an
/// undetectable hole in an otherwise fluent stitched transcript.
private func transcribeChunkWithRetry(
samples: [Float],
chunkIndex: Int
) async -> ASRChunkResult {
let first = await transcribeChunk(samples: samples)
guard case .failure(let message) = first else { return first }
guard !cancelled, !Task.isCancelled else { return .cancelled }
FlowTrace.warn(
"pipeline.chunk.retry",
"chunk=\(chunkIndex) samples=\(samples.count) error=\(message)"
)
do {
try await Task.sleep(nanoseconds: 150_000_000)
} catch {
return .cancelled
}
guard !cancelled, !Task.isCancelled else { return .cancelled }
return await transcribeChunk(samples: samples)
}
/// Pairs each chunk's audio with the text it produced, so an empty
/// transcript can be attributed to either silent audio or a mute engine.
private func logChunkOutcome(chunk: UtteranceAudioChunk, result: ASRChunkResult) {
let audio = "chunk=\(chunk.index) samples=\(chunk.samples.count) "
+ "seconds=\(FlowTrace.seconds(samples: chunk.samples.count, sampleRate: config.sampleRate)) "
+ "rms=\(FlowTrace.rms(chunk.samples)) isLast=\(chunk.isLast ? 1 : 0)"
switch result {
case .success(let text):
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty {
FlowTrace.warn("pipeline.chunk.emptyText", audio)
} else {
FlowTrace.transcript("asr.chunk", trimmed, audio)
}
case .failure(let message):
FlowTrace.warn("pipeline.chunk.failed", "\(audio) error=\(message)")
case .cancelled:
FlowTrace.pipeline("chunk.cancelled", audio)
}
}
private func publishPartial(
from stitcher: UtteranceTranscriptStitcher,
onPartial: @Sendable (String) -> Void
) {
let partial = stitcher.composedSafely()
if !partial.isEmpty {
onPartial(partial)
}
}
}
@@ -0,0 +1,172 @@
// AlibabaVocabularySync.swift
// OSGKeyboard · Shared
//
// Syncs PersonalDictionary DashScope custom vocabulary (Fun-ASR Flash).
import Foundation
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
public enum AlibabaVocabularySync {
public enum Keys {
public static let vocabularyId = "config.alibabaASRVocabularyId"
public static let fingerprint = "config.alibabaASRVocabularyFingerprint"
}
private static let vocabularyPrefix = "osgkb"
/// Returns a ready `vocabulary_id`, creating or updating the remote list as needed.
public static func ensureVocabularyID(
dictionary: PersonalDictionary,
apiKey: String,
targetModel: String = CloudASRModelCatalog.alibabaVocabularyTargetModel,
defaults: UserDefaults,
session: URLSession = .shared
) async throws -> String? {
let entries = dictionary.alibabaHotwordEntries()
guard !entries.isEmpty else {
clearCache(defaults: defaults)
return nil
}
let fingerprint = dictionary.vocabularySyncFingerprint()
if let cachedID = defaults.string(forKey: Keys.vocabularyId),
defaults.string(forKey: Keys.fingerprint) == fingerprint,
!cachedID.isEmpty {
return cachedID
}
let url = try customizationURL()
if let existingID = defaults.string(forKey: Keys.vocabularyId), !existingID.isEmpty {
try await updateVocabulary(
id: existingID,
entries: entries,
apiKey: apiKey,
url: url,
session: session
)
cache(id: existingID, fingerprint: fingerprint, defaults: defaults)
return existingID
}
let createdID = try await createVocabulary(
entries: entries,
targetModel: targetModel,
apiKey: apiKey,
url: url,
session: session
)
cache(id: createdID, fingerprint: fingerprint, defaults: defaults)
return createdID
}
public static func clearCache(defaults: UserDefaults) {
defaults.removeObject(forKey: Keys.vocabularyId)
defaults.removeObject(forKey: Keys.fingerprint)
}
private static func cache(id: String, fingerprint: String, defaults: UserDefaults) {
defaults.set(id, forKey: Keys.vocabularyId)
defaults.set(fingerprint, forKey: Keys.fingerprint)
}
private static func customizationURL() throws -> URL {
let raw = CloudASRModelCatalog.alibabaAPIBase + CloudASRModelCatalog.alibabaCustomizationPath
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
return url
}
private static func createVocabulary(
entries: [AlibabaHotwordEntry],
targetModel: String,
apiKey: String,
url: URL,
session: URLSession
) async throws -> String {
let vocabulary = entries.map { entry -> [String: Any] in
var item: [String: Any] = ["text": entry.text, "weight": entry.weight]
if let lang = entry.lang { item["lang"] = lang }
return item
}
let body: [String: Any] = [
"model": "speech-biasing",
"input": [
"action": "create_vocabulary",
"target_model": targetModel,
"prefix": vocabularyPrefix,
"vocabulary": vocabulary,
] as [String: Any],
]
let data = try await postJSON(body, to: url, apiKey: apiKey, session: session)
guard let id = parseVocabularyID(from: data) else {
throw CloudASRError.decoding("missing vocabulary_id")
}
return id
}
private static func updateVocabulary(
id: String,
entries: [AlibabaHotwordEntry],
apiKey: String,
url: URL,
session: URLSession
) async throws {
let vocabulary = entries.map { entry -> [String: Any] in
var item: [String: Any] = ["text": entry.text, "weight": entry.weight]
if let lang = entry.lang { item["lang"] = lang }
return item
}
let body: [String: Any] = [
"model": "speech-biasing",
"input": [
"action": "update_vocabulary",
"vocabulary_id": id,
"vocabulary": vocabulary,
] as [String: Any],
]
_ = try await postJSON(body, to: url, apiKey: apiKey, session: session)
}
private static func postJSON(
_ body: [String: Any],
to url: URL,
apiKey: String,
session: URLSession
) async throws -> [String: Any] {
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
request.httpBody = try JSONSerialization.data(withJSONObject: body)
let (data, response) = try await session.data(for: request)
guard let http = response as? HTTPURLResponse else {
throw CloudASRError.transport("non-HTTP response")
}
let json = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any]
guard (200..<300).contains(http.statusCode) else {
let message = parseAPIErrorMessage(from: json)
throw CloudASRError.http(status: http.statusCode, message: message)
}
return json ?? [:]
}
private static func parseVocabularyID(from json: [String: Any]) -> String? {
if let output = json["output"] as? [String: Any],
let id = output["vocabulary_id"] as? String {
return id
}
return nil
}
private static func parseAPIErrorMessage(from json: [String: Any]?) -> String? {
guard let json else { return nil }
if let message = json["message"] as? String { return message }
if let error = json["error"] as? [String: Any],
let message = error["message"] as? String {
return message
}
return nil
}
}
@@ -0,0 +1,430 @@
// BailianRealtimeASRClient.swift
// OSGKeyboard · Shared
//
// Alibaba Cloud Bailian / DashScope realtime ASR over the classic inference
// WebSocket (`/api-ws/v1/inference`). Utterance-level duplex session with
// interim `result-generated` partials; batch `transcribe(samples:)` remains
// for connection probes and chunk fallback.
import Foundation
import os
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
struct BailianRealtimeASRClient: CloudASRTranscribing, CloudASRStreamingCapable {
let apiKey: String
let endpoint: String
let model: String
let vocabularyID: String?
let session: URLSession
/// 100 ms of 16 kHz / 16-bit / mono PCM.
static let targetChunkBytes = 3_200
static let startTimeout: TimeInterval = 8
static let finalTimeout: TimeInterval = 12
private static let sessionTimeout: TimeInterval = startTimeout + finalTimeout + 4
func prepare(dictionary: PersonalDictionary) async throws {}
func openStreamingSession(
locale: Locale,
dictionary: PersonalDictionary,
onPartial: @escaping @Sendable (String) -> Void
) async throws -> any CloudASRStreamingSession {
_ = locale
_ = dictionary
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
let url = try resolvedEndpointURL()
let resolvedModel = model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
? CloudASRModelCatalog.alibabaFunASRRealtime
: model.trimmingCharacters(in: .whitespacesAndNewlines)
var request = URLRequest(url: url)
request.timeoutInterval = 8
request.setValue(
"bearer \(apiKey.trimmingCharacters(in: .whitespacesAndNewlines))",
forHTTPHeaderField: "Authorization"
)
let wsTask = session.webSocketTask(with: request)
wsTask.resume()
let live = BailianStreamingSession(
wsTask: wsTask,
model: resolvedModel,
vocabularyID: vocabularyID,
onPartial: onPartial
)
try await live.start()
return live
}
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String {
guard sampleRate == 16_000 else {
throw CloudASRError.transport("Bailian realtime expects 16 kHz audio")
}
guard !samples.isEmpty else { throw CloudASRError.emptyTranscript }
let session = try await openStreamingSession(
locale: locale,
dictionary: dictionary,
onPartial: { _ in }
)
try await session.append(samples: samples)
let text = try await session.finish()
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { throw CloudASRError.emptyTranscript }
return trimmed
}
/// Settings connection probe: handshake to `task-started` only.
func probeConnection() async throws {
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
let url = try resolvedEndpointURL()
let taskID = UUID().uuidString.replacingOccurrences(of: "-", with: "")
let resolvedModel = model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
? CloudASRModelCatalog.alibabaFunASRRealtime
: model.trimmingCharacters(in: .whitespacesAndNewlines)
var request = URLRequest(url: url)
request.timeoutInterval = 8
request.setValue(
"bearer \(apiKey.trimmingCharacters(in: .whitespacesAndNewlines))",
forHTTPHeaderField: "Authorization"
)
let wsTask = session.webSocketTask(with: request)
wsTask.resume()
try await withThrowingTaskGroup(of: Void.self) { group in
let events = BailianEventStream(task: wsTask, onPartial: nil)
group.addTask {
defer { events.cancel() }
try await BailianRealtimeASRClient.sendText(
BailianRealtimeASRClient.runTaskMessage(
taskID: taskID,
model: resolvedModel,
vocabularyID: nil
),
task: wsTask
)
try await events.waitForStarted(timeout: Self.startTimeout)
try? await BailianRealtimeASRClient.sendText(
BailianRealtimeASRClient.finishTaskMessage(taskID: taskID),
task: wsTask
)
}
group.addTask {
try await Task.sleep(nanoseconds: UInt64(Self.startTimeout * 1_000_000_000))
events.cancel()
wsTask.cancel(with: .goingAway, reason: nil)
throw CloudASRError.transport("connection probe timed out")
}
_ = try await group.next()
group.cancelAll()
}
}
private func resolvedEndpointURL() throws -> URL {
let raw = endpoint.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
? CloudASRModelCatalog.bailianDefaultEndpoint
: endpoint.trimmingCharacters(in: .whitespacesAndNewlines)
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
return url
}
static func sendText(_ text: String, task: URLSessionWebSocketTask) async throws {
do {
try await task.send(.string(text))
} catch {
throw CloudASRError.transport(error.localizedDescription)
}
}
static func sendBinary(_ data: Data, task: URLSessionWebSocketTask) async throws {
do {
try await task.send(.data(data))
} catch {
throw CloudASRError.transport(error.localizedDescription)
}
}
/// Overlap-aware join to avoid cumulative duplicate text from interim replays.
static func mergeSegments(_ segments: [String]) -> String {
var result = ""
for segment in segments {
if result.isEmpty {
result = segment
continue
}
let resultChars = Array(result)
let segmentChars = Array(segment)
let maxOverlap = min(resultChars.count, segmentChars.count)
var overlap = 0
if maxOverlap >= 2 {
for length in stride(from: maxOverlap, through: 2, by: -1) {
let tail = resultChars.suffix(length)
let head = segmentChars.prefix(length)
if tail.elementsEqual(head) {
overlap = length
break
}
}
}
result.append(contentsOf: segmentChars.dropFirst(overlap))
}
return result
}
static func runTaskMessage(taskID: String, model: String, vocabularyID: String?) -> String {
var parameters: [String: Any] = [
"sample_rate": 16_000,
"format": "pcm",
]
if let vocabularyID = vocabularyID?.trimmingCharacters(in: .whitespacesAndNewlines),
!vocabularyID.isEmpty {
parameters["vocabulary_id"] = vocabularyID
}
let body: [String: Any] = [
"header": [
"action": "run-task",
"task_id": taskID,
"streaming": "duplex",
],
"payload": [
"task_group": "audio",
"task": "asr",
"function": "recognition",
"model": model,
"parameters": parameters,
"input": [:] as [String: Any],
],
]
guard let data = try? JSONSerialization.data(withJSONObject: body),
let json = String(data: data, encoding: .utf8) else {
return "{}"
}
return json
}
static func finishTaskMessage(taskID: String) -> String {
let body: [String: Any] = [
"header": [
"action": "finish-task",
"task_id": taskID,
"streaming": "duplex",
],
"payload": ["input": [:] as [String: Any]],
]
guard let data = try? JSONSerialization.data(withJSONObject: body),
let json = String(data: data, encoding: .utf8) else {
return "{}"
}
return json
}
}
// MARK: - Utterance session
private final class BailianStreamingSession: CloudASRStreamingSession, @unchecked Sendable {
private let wsTask: URLSessionWebSocketTask
private let model: String
private let vocabularyID: String?
private let onPartial: @Sendable (String) -> Void
private let events: BailianEventStream
private let taskID: String
private let lock = OSAllocatedUnfairLock()
private var started = false
private var pcmBuffer = Data()
init(
wsTask: URLSessionWebSocketTask,
model: String,
vocabularyID: String?,
onPartial: @escaping @Sendable (String) -> Void
) {
self.wsTask = wsTask
self.model = model
self.vocabularyID = vocabularyID
self.onPartial = onPartial
self.taskID = UUID().uuidString.replacingOccurrences(of: "-", with: "")
self.events = BailianEventStream(task: wsTask, onPartial: onPartial)
}
func start() async throws {
try await BailianRealtimeASRClient.sendText(
BailianRealtimeASRClient.runTaskMessage(
taskID: taskID,
model: model,
vocabularyID: vocabularyID
),
task: wsTask
)
try await events.waitForStarted(timeout: BailianRealtimeASRClient.startTimeout)
lock.withLock { started = true }
}
func append(samples: [Float]) async throws {
guard lock.withLock({ started }) else {
throw CloudASRError.transport("Bailian session not started")
}
let pcm = CloudASRStreamingPCM.pcm16LE(samples: samples)
let frames: [Data] = lock.withLock {
pcmBuffer.append(pcm)
var frames: [Data] = []
while pcmBuffer.count >= BailianRealtimeASRClient.targetChunkBytes {
let frame = pcmBuffer.prefix(BailianRealtimeASRClient.targetChunkBytes)
frames.append(Data(frame))
pcmBuffer.removeFirst(BailianRealtimeASRClient.targetChunkBytes)
}
return frames
}
for frame in frames {
try await BailianRealtimeASRClient.sendBinary(frame, task: wsTask)
}
}
func finish() async throws -> String {
// Flush remaining PCM (pad short last frame as-is server tolerates).
let trailing: Data = lock.withLock {
let data = pcmBuffer
pcmBuffer.removeAll(keepingCapacity: false)
return data
}
if !trailing.isEmpty {
try await BailianRealtimeASRClient.sendBinary(trailing, task: wsTask)
}
// Avoid emptyAudio race on very short clips.
try? await Task.sleep(nanoseconds: 120_000_000)
try await BailianRealtimeASRClient.sendText(
BailianRealtimeASRClient.finishTaskMessage(taskID: taskID),
task: wsTask
)
return try await events.waitForFinalText(timeout: BailianRealtimeASRClient.finalTimeout)
}
func cancel() {
events.cancel()
}
}
// MARK: - Concurrent read loop
private final class BailianEventStream: @unchecked Sendable {
private let task: URLSessionWebSocketTask
private let onPartial: (@Sendable (String) -> Void)?
private let lock = OSAllocatedUnfairLock()
private var started = false
private var finalText: String?
private var failure: Error?
private var readTask: Task<Void, Never>?
init(task: URLSessionWebSocketTask, onPartial: (@Sendable (String) -> Void)?) {
self.task = task
self.onPartial = onPartial
readTask = Task { [weak self] in
await self?.readLoop()
}
}
func cancel() {
readTask?.cancel()
task.cancel(with: .goingAway, reason: nil)
}
func waitForStarted(timeout: TimeInterval) async throws {
let deadline = Date().addingTimeInterval(timeout)
while Date() < deadline {
if let failure = snapshotFailure() { throw failure }
if snapshotStarted() { return }
try await Task.sleep(nanoseconds: 20_000_000)
}
cancel()
throw CloudASRError.transport("task-started timed out")
}
func waitForFinalText(timeout: TimeInterval) async throws -> String {
let deadline = Date().addingTimeInterval(timeout)
while Date() < deadline {
if let failure = snapshotFailure() { throw failure }
if let text = snapshotFinalText() { return text }
try await Task.sleep(nanoseconds: 20_000_000)
}
cancel()
throw CloudASRError.transport("final result timed out")
}
private func snapshotStarted() -> Bool {
lock.withLock { started }
}
private func snapshotFinalText() -> String? {
lock.withLock { finalText }
}
private func snapshotFailure() -> Error? {
lock.withLock { failure }
}
private func readLoop() async {
var reducer = BailianASREventReducer()
while !Task.isCancelled {
let message: URLSessionWebSocketTask.Message
do {
message = try await task.receive()
} catch {
publishFailure(CloudASRError.transport(error.localizedDescription))
return
}
let text: String
switch message {
case .string(let value):
text = value
case .data(let data):
text = String(data: data, encoding: .utf8) ?? ""
@unknown default:
continue
}
guard !text.isEmpty else { continue }
switch reducer.apply(jsonText: text) {
case .none:
continue
case .started:
publishStarted()
case .partial(let display):
onPartial?(display)
case .finished(let final):
publishFinal(final)
return
case .failed(let message):
publishFailure(CloudASRError.transport(message))
return
}
}
}
private func publishStarted() {
lock.withLock { started = true }
}
private func publishFinal(_ text: String) {
lock.withLock { finalText = text }
}
private func publishFailure(_ error: Error) {
lock.withLock { failure = error }
cancel()
}
}
@@ -0,0 +1,526 @@
// CloudASRClients.swift
// OSGKeyboard · Shared
//
// Provider-specific cloud ASR backends with personal-dictionary bias.
import Foundation
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
public protocol CloudASRTranscribing: Sendable {
func prepare(dictionary: PersonalDictionary) async throws
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String
/// Settings "validate connection" probe. Verifies transport + auth only.
func probeConnection() async throws
}
extension CloudASRTranscribing {
/// Default probe: transcribe ~1 s of near-silence. An empty transcript
/// counts as success HTTP/streaming providers only need to prove that
/// transport + auth work. Providers that reject silent/short audio or
/// close after an empty final (DashScope realtime, Volcengine SAUC)
/// override with a handshake-only probe.
public func probeConnection() async throws {
do {
_ = try await transcribe(
samples: [Float](repeating: 0.01, count: 16_000),
sampleRate: 16_000,
locale: Locale(identifier: "zh-CN"),
dictionary: .empty
)
} catch CloudASRError.emptyTranscript {
return
}
}
}
public enum CloudASRClientFactory {
public static func make(store: any ConfigurationStore, session: URLSession = .shared) -> CloudASRTranscribing {
let providerId = store.asrProviderId
let strategy = CloudASRModelCatalog.strategy(for: providerId)
let asrModel = store.asrModel.isEmpty
? CloudASRModelCatalog.defaultModel(for: providerId)
: store.asrModel
switch strategy {
case .zhipuHotwords:
return ZhipuCloudASRClient(
apiKey: store.asrApiKey,
model: asrModel,
session: session
)
case .bailianStreaming:
return BailianRealtimeASRClient(
apiKey: store.asrApiKey,
endpoint: store.asrBaseURL,
model: asrModel,
vocabularyID: nil,
session: session
)
case .prompt:
return PromptCloudASRClient(
providerId: providerId,
baseURL: store.asrBaseURL,
apiKey: store.asrApiKey,
model: asrModel,
session: session
)
case .openRouterJson:
return PromptCloudASRClient(
providerId: providerId,
baseURL: store.asrBaseURL,
apiKey: store.asrApiKey,
model: asrModel,
session: session,
requestFormat: .openRouterJson
)
case .volcengineStreaming:
return VolcengineCloudASRClient(
apiKey: store.asrApiKey,
endpoint: store.asrBaseURL,
resourceID: asrModel,
session: session
)
case .openaiRealtimeStreaming:
return OpenAIRealtimeASRClient(
apiKey: store.asrApiKey,
endpoint: store.asrBaseURL,
model: asrModel,
batchBaseURL: LLMProvider.provider(id: "openai").defaultBaseURL,
session: session
)
case .localFallback:
return UnsupportedCloudASRClient(providerId: providerId)
}
}
}
// MARK: - Zhipu (hotwords + prompt)
struct ZhipuCloudASRClient: CloudASRTranscribing {
let apiKey: String
let model: String
let session: URLSession
private static let maxDurationSeconds: TimeInterval = 30
func prepare(dictionary: PersonalDictionary) async throws {}
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String {
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
let duration = Double(samples.count) / Double(sampleRate)
guard duration <= Self.maxDurationSeconds else { throw CloudASRError.audioTooLong }
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
let urlString = "https://open.bigmodel.cn/api/paas/v4\(CloudASRModelCatalog.zhipuTranscriptionPath)"
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
let boundary = "Boundary-\(UUID().uuidString)"
var body = Data()
func appendField(_ name: String, _ value: String) {
body.append("--\(boundary)\r\n".data(using: .utf8)!)
body.append("Content-Disposition: form-data; name=\"\(name)\"\r\n\r\n".data(using: .utf8)!)
body.append("\(value)\r\n".data(using: .utf8)!)
}
appendField("model", model)
appendField("stream", "false")
let hotwords = dictionary.asrHotwords()
if !hotwords.isEmpty,
let hotwordsJSON = try? JSONSerialization.data(withJSONObject: hotwords),
let hotwordsString = String(data: hotwordsJSON, encoding: .utf8) {
appendField("hotwords", hotwordsString)
}
let prompt = dictionary.asrPromptBias()
if !prompt.isEmpty {
appendField("prompt", prompt)
}
body.append("--\(boundary)\r\n".data(using: .utf8)!)
body.append("Content-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\n".data(using: .utf8)!)
body.append("Content-Type: audio/wav\r\n\r\n".data(using: .utf8)!)
body.append(wav)
body.append("\r\n".data(using: .utf8)!)
body.append("--\(boundary)--\r\n".data(using: .utf8)!)
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")
request.httpBody = body
request.timeoutInterval = 60
let (data, response) = try await session.data(for: request)
try Self.validateHTTP(response: response, data: data)
guard let text = Self.parseZhipuText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
!text.isEmpty else {
throw CloudASRError.emptyTranscript
}
return text
}
private static func parseZhipuText(from data: Data) -> String? {
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
return nil
}
return json["text"] as? String
}
fileprivate static func validateHTTP(response: URLResponse, data: Data) throws {
guard let http = response as? HTTPURLResponse else {
throw CloudASRError.transport("non-HTTP response")
}
guard (200..<300).contains(http.statusCode) else {
let message = parseErrorMessage(from: data)
throw CloudASRError.http(status: http.statusCode, message: message)
}
}
fileprivate static func parseErrorMessage(from data: Data) -> String? {
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
return nil
}
if let message = json["message"] as? String { return message }
if let error = json["error"] as? [String: Any],
let message = error["message"] as? String {
return message
}
return nil
}
}
// MARK: - Alibaba Fun-ASR Flash (HTTP sync, context text bias)
struct AlibabaFunASRClient: CloudASRTranscribing {
let apiKey: String
let model: String
let session: URLSession
func prepare(dictionary: PersonalDictionary) async throws {}
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String {
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
let dataURI = PCMSampleWavEncoder.dataURI(samples: samples, sampleRate: sampleRate)
let urlString = CloudASRModelCatalog.alibabaAPIBase + CloudASRModelCatalog.alibabaMultimodalPath
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
var messages: [[String: Any]] = []
let context = dictionary.alibabaContextText()
if !context.isEmpty {
messages.append([
"role": "user",
"content": [
["type": "input_text", "text": context],
],
])
}
messages.append([
"role": "user",
"content": [
[
"type": "input_audio",
"input_audio": ["data": dataURI],
],
],
])
let parameters: [String: Any] = [
"format": "wav",
"sample_rate": "\(sampleRate)",
]
let body: [String: Any] = [
"model": model,
"input": ["messages": messages],
"parameters": parameters,
]
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
request.setValue("disable", forHTTPHeaderField: "X-DashScope-SSE")
request.httpBody = try JSONSerialization.data(withJSONObject: body)
request.timeoutInterval = 90
let (data, response) = try await session.data(for: request)
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
if let text = Self.parseText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
!text.isEmpty {
return text
}
return ""
}
private static func parseText(from data: Data) -> String? {
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
let output = json["output"] as? [String: Any] else {
return nil
}
if let text = output["text"] as? String { return text }
if let sentence = output["sentence"] as? [String: Any],
let text = sentence["text"] as? String {
return text
}
return nil
}
}
// MARK: - Prompt-biased transcription (OpenAI / MiMo / Groq / custom)
enum PromptCloudASRRequestFormat: Sendable {
case multipart
/// OpenRouter expects JSON `{ model, input_audio: { data, format } }`.
case openRouterJson
}
struct PromptCloudASRClient: CloudASRTranscribing {
let providerId: String
let baseURL: String
let apiKey: String
let model: String
let session: URLSession
var requestFormat: PromptCloudASRRequestFormat = .multipart
/// Groq / OpenRouter batch uploads cap around 30 s per request.
private static let whisperCompatibleMaxDurationSeconds: TimeInterval = 30
init(
providerId: String,
baseURL: String,
apiKey: String,
model: String,
session: URLSession,
requestFormat: PromptCloudASRRequestFormat = .multipart
) {
self.providerId = providerId
self.baseURL = baseURL
self.apiKey = apiKey
self.model = model
self.session = session
self.requestFormat = requestFormat
}
func prepare(dictionary: PersonalDictionary) async throws {}
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String {
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
if providerId == "mimo" {
return try await transcribeMiMo(
samples: samples,
sampleRate: sampleRate,
dictionary: dictionary
)
}
if requestFormat == .openRouterJson {
return try await transcribeOpenRouterJSON(
samples: samples,
sampleRate: sampleRate,
dictionary: dictionary
)
}
return try await transcribeOpenAIStyle(
samples: samples,
sampleRate: sampleRate,
dictionary: dictionary
)
}
private func enforceWhisperDuration(samples: [Float], sampleRate: Int) throws {
let duration = Double(samples.count) / Double(sampleRate)
guard duration <= Self.whisperCompatibleMaxDurationSeconds else {
throw CloudASRError.audioTooLong
}
}
private func transcribeOpenAIStyle(
samples: [Float],
sampleRate: Int,
dictionary: PersonalDictionary
) async throws -> String {
if providerId == "groq" || providerId == "openai" || providerId == "custom" {
try enforceWhisperDuration(samples: samples, sampleRate: sampleRate)
}
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
let urlString = "\(trimmedBase)/audio/transcriptions"
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
let boundary = "Boundary-\(UUID().uuidString)"
var body = Data()
func appendField(_ name: String, _ value: String) {
body.append("--\(boundary)\r\n".data(using: .utf8)!)
body.append("Content-Disposition: form-data; name=\"\(name)\"\r\n\r\n".data(using: .utf8)!)
body.append("\(value)\r\n".data(using: .utf8)!)
}
appendField("model", model)
let prompt = dictionary.asrPromptBias(maxCharacters: 600)
if !prompt.isEmpty {
appendField("prompt", prompt)
}
body.append("--\(boundary)\r\n".data(using: .utf8)!)
body.append("Content-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\n".data(using: .utf8)!)
body.append("Content-Type: audio/wav\r\n\r\n".data(using: .utf8)!)
body.append(wav)
body.append("\r\n".data(using: .utf8)!)
body.append("--\(boundary)--\r\n".data(using: .utf8)!)
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")
request.httpBody = body
request.timeoutInterval = 90
let (data, response) = try await session.data(for: request)
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
guard let text = Self.parseOpenAIText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
!text.isEmpty else {
throw CloudASRError.emptyTranscript
}
return text
}
private func transcribeOpenRouterJSON(
samples: [Float],
sampleRate: Int,
dictionary: PersonalDictionary
) async throws -> String {
try enforceWhisperDuration(samples: samples, sampleRate: sampleRate)
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
let urlString = "\(trimmedBase)/audio/transcriptions"
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
var body: [String: Any] = [
"model": model,
"input_audio": [
"data": wav.base64EncodedString(),
"format": "wav",
],
]
let prompt = dictionary.asrPromptBias(maxCharacters: 600)
if !prompt.isEmpty {
body["prompt"] = prompt
}
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
request.httpBody = try JSONSerialization.data(withJSONObject: body)
request.timeoutInterval = 90
let (data, response) = try await session.data(for: request)
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
guard let text = Self.parseOpenAIText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
!text.isEmpty else {
throw CloudASRError.emptyTranscript
}
return text
}
private func transcribeMiMo(
samples: [Float],
sampleRate: Int,
dictionary: PersonalDictionary
) async throws -> String {
let dataURI = PCMSampleWavEncoder.dataURI(samples: samples, sampleRate: sampleRate)
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
let urlString = "\(trimmedBase)/chat/completions"
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
var userContent: [[String: Any]] = []
let prompt = dictionary.asrPromptBias()
if !prompt.isEmpty {
userContent.append(["type": "text", "text": prompt])
}
userContent.append([
"type": "input_audio",
"input_audio": ["data": dataURI],
])
let body: [String: Any] = [
"model": model,
"messages": [
["role": "user", "content": userContent],
],
"asr_options": ["language": "auto"],
]
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
request.setValue(apiKey, forHTTPHeaderField: "api-key")
request.httpBody = try JSONSerialization.data(withJSONObject: body)
request.timeoutInterval = 90
let (data, response) = try await session.data(for: request)
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
guard let text = Self.parseChatCompletionText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
!text.isEmpty else {
throw CloudASRError.emptyTranscript
}
return text
}
private static func parseOpenAIText(from data: Data) -> String? {
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
return nil
}
return json["text"] as? String
}
private static func parseChatCompletionText(from data: Data) -> String? {
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
let choices = json["choices"] as? [[String: Any]],
let first = choices.first,
let message = first["message"] as? [String: Any] else {
return nil
}
return message["content"] as? String
}
}
// MARK: - Unsupported hosted ASR (Moonshot)
struct UnsupportedCloudASRClient: CloudASRTranscribing {
let providerId: String
func prepare(dictionary: PersonalDictionary) async throws {}
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String {
throw CloudASRError.providerUnsupported
}
}
@@ -0,0 +1,22 @@
// CloudASRConnectionCheck.swift
// OSGKeyboard · Shared
//
// Settings "validate connection" probe shared by iOS and macOS.
import Foundation
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
public enum CloudASRConnectionCheck {
/// Verifies the active cloud ASR client can connect + authenticate.
///
/// Each backend decides how to probe (see `CloudASRTranscribing`):
/// HTTP/batch providers transcribe a short silence clip and treat an
/// empty transcript as success; Bailian / Volcengine / OpenAI Realtime
/// handshake (and auth) only silence clips make streaming backends fail.
public static func validate(store: any ConfigurationStore) async throws {
let client = CloudASRClientFactory.make(store: store)
try await client.probeConnection()
}
}
@@ -0,0 +1,250 @@
// CloudASRService.swift
// OSGKeyboard · Shared
//
// Cloud-engine ASR: uploads PCM to the user's configured provider with
// personal-dictionary bias. Streaming-capable providers use one utterance
// WebSocket; others stay on chunked batch. Moonshot falls back to on-device ASR.
import Foundation
import os
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
public final class CloudASRService: ASRService, @unchecked Sendable {
private let store: any ConfigurationStore
private let session: URLSession
private let localFallback: ASRService
private let lock = OSAllocatedUnfairLock()
private var client: CloudASRTranscribing?
private var usesLocalFallback = false
private var boundProviderId: String?
private var cancelled = false
private var streamingPipeline: StreamingUtterancePipeline?
public init(
store: any ConfigurationStore = AppGroupStore(),
session: URLSession = .shared,
localFallback: ASRService? = nil
) {
self.store = store
self.session = session
// `SpeechAnalyzerASR` is internal, so it can't appear in a public
// default argument value resolve the fallback in the body instead.
self.localFallback = localFallback ?? SpeechAnalyzerASR()
}
/// Whether Flow should prefer utterance-level true streaming for the bound provider.
public var supportsUtteranceStreaming: Bool {
CloudASRModelCatalog.supportsTrueStreamingASR(for: store.asrProviderId)
}
public func resetForNewUtterance() {
lock.withLock { cancelled = false }
if usesLocalFallback {
localFallback.resetForNewUtterance()
}
}
public func warmup(locale: Locale) async {
bindClientIfNeeded()
if usesLocalFallback {
await localFallback.warmup(locale: locale)
return
}
guard let client = lock.withLock({ client }) else { return }
do {
try await client.prepare(dictionary: store.personalDictionary)
} catch {
OSGLog.asr.warning("cloud ASR vocabulary prepare failed: \(error.localizedDescription, privacy: .public)")
}
}
public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
guard !samples.isEmpty else { return .success("") }
if Task.isCancelled || lock.withLock({ cancelled }) { return .cancelled }
bindClientIfNeeded()
if usesLocalFallback {
return await localFallback.transcribeChunk(samples: samples, locale: locale)
}
guard let client = lock.withLock({ client }) else {
return .failure(CloudASRError.providerUnsupported.localizedDescription)
}
let startedAt = Date()
do {
let text = try await client.transcribe(
samples: samples,
sampleRate: 16_000,
locale: locale,
dictionary: store.personalDictionary
)
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
FlowTrace.transcript(
"asr.cloud.chunk",
trimmed,
"engine=cloud provider=\(store.asrProviderId) samples=\(samples.count) "
+ "rms=\(FlowTrace.rms(samples)) elapsed=\(FlowTrace.seconds(since: startedAt))s"
)
return trimmed.isEmpty ? .success("") : .success(trimmed)
} catch is CancellationError {
FlowTrace.asr("cloud.chunk.cancelled", "samples=\(samples.count)")
return .cancelled
} catch {
FlowTrace.warn(
"asr.cloud.chunk.failed",
"provider=\(store.asrProviderId) samples=\(samples.count) "
+ "rms=\(FlowTrace.rms(samples)) elapsed=\(FlowTrace.seconds(since: startedAt))s "
+ "error=\(error.localizedDescription)"
)
return .failure(error.localizedDescription)
}
}
/// Utterance-level streaming; if the session cannot start, fall back to
/// chunked batch on the same mic stream. Mid-stream failures surface as
/// errors (finalize still has PCM batch fallback).
public func transcribeUtteranceStreaming(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale,
onPartial: @escaping @Sendable (String) -> Void
) async -> ChunkedUtterancePipelineOutcome {
bindClientIfNeeded()
if usesLocalFallback {
let pipeline = ChunkedUtterancePipeline(asr: localFallback, locale: locale)
return await pipeline.transcribe(stream: stream, onPartial: onPartial)
}
guard let streamingClient = lock.withLock({ client as? CloudASRStreamingCapable }) else {
let pipeline = ChunkedUtterancePipeline(asr: self, locale: locale)
return await pipeline.transcribe(stream: stream, onPartial: onPartial)
}
let session: any CloudASRStreamingSession
do {
session = try await streamingClient.openStreamingSession(
locale: locale,
dictionary: store.personalDictionary,
onPartial: onPartial
)
} catch {
OSGLog.asr.warning(
"streaming ASR session open failed, using chunked batch: \(error.localizedDescription, privacy: .public)"
)
let pipeline = ChunkedUtterancePipeline(asr: self, locale: locale)
return await pipeline.transcribe(stream: stream, onPartial: onPartial)
}
let pipeline = StreamingUtterancePipeline(
client: streamingClient,
locale: locale,
dictionary: store.personalDictionary
)
lock.withLock { streamingPipeline = pipeline }
let outcome = await pipeline.transcribe(
stream: stream,
onPartial: onPartial,
preopenedSession: session
)
lock.withLock { streamingPipeline = nil }
return outcome
}
public func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent> {
bindClientIfNeeded()
if usesLocalFallback {
return localFallback.transcribe(stream: stream, locale: locale)
}
if supportsUtteranceStreaming, lock.withLock({ client is CloudASRStreamingCapable }) {
return AsyncStream { continuation in
continuation.yield(.capability(onDeviceSupported: false))
let task = Task {
let outcome = await self.transcribeUtteranceStreaming(
stream: stream,
locale: locale,
onPartial: { partial in
continuation.yield(.partial(partial))
}
)
switch outcome {
case .success(let success):
continuation.yield(.final(success.text))
case .failure(let message):
continuation.yield(.error(message))
case .cancelled:
break
}
continuation.finish()
}
continuation.onTermination = { @Sendable _ in
task.cancel()
self.cancel()
}
}
}
return AsyncStream { continuation in
continuation.yield(.capability(onDeviceSupported: false))
let task = Task {
var samples: [Float] = []
for await snap in stream {
if Task.isCancelled { break }
samples.append(contentsOf: snap.samples)
}
guard !Task.isCancelled, !self.lock.withLock({ self.cancelled }) else {
continuation.finish()
return
}
guard !samples.isEmpty else {
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
continuation.finish()
return
}
switch await self.transcribeChunk(samples: samples, locale: locale) {
case .success(let text):
if text.isEmpty {
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
} else {
continuation.yield(.final(text))
}
case .failure(let message):
continuation.yield(.error(message))
case .cancelled:
break
}
continuation.finish()
}
continuation.onTermination = { @Sendable _ in
task.cancel()
self.cancel()
}
}
}
public func cancel() {
lock.withLock { cancelled = true }
let pipeline = lock.withLock { streamingPipeline }
Task { await pipeline?.cancel() }
localFallback.cancel()
}
private func bindClientIfNeeded() {
let providerId = store.asrProviderId
let strategy = CloudASRModelCatalog.strategy(for: providerId)
lock.withLock {
guard boundProviderId != providerId else { return }
boundProviderId = providerId
usesLocalFallback = strategy == .localFallback
client = usesLocalFallback
? nil
: CloudASRClientFactory.make(store: store, session: session)
}
}
}
@@ -0,0 +1,178 @@
// CloudASRStreaming.swift
// OSGKeyboard · Shared
//
// Utterance-scoped cloud ASR sessions: one long-lived connection per press,
// streaming PCM up and interim text down. Chunked batch ASR remains the
// fallback for providers without a true streaming protocol.
import Foundation
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
/// Long-lived cloud ASR session for one Flow utterance.
public protocol CloudASRStreamingSession: Sendable {
/// Append 16 kHz mono Float32 PCM captured while the mic is open.
func append(samples: [Float]) async throws
/// Signal end-of-audio and wait for the polish-ready final transcript.
func finish() async throws -> String
func cancel()
}
/// Providers that can open an utterance-level streaming session.
public protocol CloudASRStreamingCapable: CloudASRTranscribing {
func openStreamingSession(
locale: Locale,
dictionary: PersonalDictionary,
onPartial: @escaping @Sendable (String) -> Void
) async throws -> any CloudASRStreamingSession
}
/// Feeds a live mic stream into a cloud streaming session and mirrors the
/// existing `ChunkedUtterancePipelineOutcome` surface for Flow.
public actor StreamingUtterancePipeline {
private let client: any CloudASRStreamingCapable
private let locale: Locale
private let dictionary: PersonalDictionary
private var cancelled = false
private var activeSession: (any CloudASRStreamingSession)?
public init(
client: any CloudASRStreamingCapable,
locale: Locale,
dictionary: PersonalDictionary
) {
self.client = client
self.locale = locale
self.dictionary = dictionary
}
public func cancel() {
cancelled = true
activeSession?.cancel()
}
public func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
onPartial: @Sendable @escaping (String) -> Void,
preopenedSession: (any CloudASRStreamingSession)? = nil
) async -> ChunkedUtterancePipelineOutcome {
cancelled = false
let startedAt = Date()
// Counted so an empty cloud transcript can be told apart from "we never
// uploaded any audio" the two look identical to the user.
var uploadedSamples = 0
var uploadedSnapshots = 0
do {
let session: any CloudASRStreamingSession
if let preopenedSession {
session = preopenedSession
} else {
session = try await client.openStreamingSession(
locale: locale,
dictionary: dictionary,
onPartial: onPartial
)
}
activeSession = session
FlowTrace.asr(
"cloud.stream.opened",
"locale=\(locale.identifier(.bcp47)) preopened=\(preopenedSession != nil ? 1 : 0)"
)
for await snap in stream {
if cancelled || Task.isCancelled {
session.cancel()
FlowTrace.asr(
"cloud.stream.cancelledMidUpload",
"uploadedSamples=\(uploadedSamples)"
)
return .cancelled
}
guard !snap.samples.isEmpty else { continue }
uploadedSnapshots += 1
uploadedSamples += snap.samples.count
try await session.append(samples: snap.samples)
}
FlowTrace.asr(
"cloud.stream.uploadDone",
"snapshots=\(uploadedSnapshots) samples=\(uploadedSamples) "
+ "seconds=\(FlowTrace.seconds(samples: uploadedSamples))"
)
if cancelled || Task.isCancelled {
session.cancel()
return .cancelled
}
let finalText = try await session.finish()
.trimmingCharacters(in: .whitespacesAndNewlines)
activeSession = nil
guard !finalText.isEmpty else {
FlowTrace.warn(
"asr.cloud.stream.emptyFinal",
"uploadedSamples=\(uploadedSamples) "
+ "seconds=\(FlowTrace.seconds(samples: uploadedSamples)) "
+ "elapsed=\(FlowTrace.seconds(since: startedAt))s"
)
return .failure(SharedL10n.string("error.asr.noSpeech"))
}
FlowTrace.transcript(
"asr.cloud.final",
finalText,
"engine=cloud uploadedSamples=\(uploadedSamples) "
+ "elapsed=\(FlowTrace.seconds(since: startedAt))s"
)
return .success(ChunkedUtteranceSuccess(text: finalText))
} catch is CancellationError {
activeSession?.cancel()
activeSession = nil
FlowTrace.asr("cloud.stream.cancelled", "uploadedSamples=\(uploadedSamples)")
return .cancelled
} catch {
activeSession?.cancel()
activeSession = nil
if cancelled || Task.isCancelled { return .cancelled }
FlowTrace.warn(
"asr.cloud.stream.failed",
"uploadedSamples=\(uploadedSamples) "
+ "elapsed=\(FlowTrace.seconds(since: startedAt))s "
+ "error=\(error.localizedDescription)"
)
return .failure(error.localizedDescription)
}
}
}
/// Shared PCM helpers for streaming cloud clients.
enum CloudASRStreamingPCM {
static func pcm16LE(samples: [Float]) -> Data {
var data = Data()
data.reserveCapacity(samples.count * 2)
for sample in samples {
let scaled = sample * 32_767.0
let clipped = Swift.max(-32_768.0, Swift.min(32_767.0, scaled))
var littleEndian = Int16(clipped.rounded()).littleEndian
withUnsafeBytes(of: &littleEndian) { data.append(contentsOf: $0) }
}
return data
}
/// Linear upsample 16 kHz 24 kHz for OpenAI Realtime PCM input.
static func upsample16kTo24k(_ samples: [Float]) -> [Float] {
guard !samples.isEmpty else { return [] }
let outCount = max(1, samples.count * 3 / 2)
var output = [Float]()
output.reserveCapacity(outCount)
let lastIndex = samples.count - 1
for i in 0..<outCount {
let src = Double(i) * 16.0 / 24.0
let i0 = min(Int(src), lastIndex)
let i1 = min(i0 + 1, lastIndex)
let frac = Float(src - Double(i0))
output.append(samples[i0] + (samples[i1] - samples[i0]) * frac)
}
return output
}
}
@@ -0,0 +1,188 @@
// CloudASRStreamingEventParsing.swift
// OSGKeyboard · HostSupport
//
// Pure reducers for streaming ASR WebSocket event JSON hermetic fixtures
// without a live socket.
import Foundation
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
// MARK: - Bailian Fun-ASR Realtime
enum BailianASREventEffect: Equatable, Sendable {
case none
case started
case partial(String)
case finished(String)
case failed(String)
}
struct BailianASREventReducer: Sendable {
private(set) var finalSegments: [Int64: String] = [:]
private(set) var partialSegments: [Int64: String] = [:]
private(set) var lastResultText = ""
private(set) var started = false
mutating func apply(jsonText: String) -> BailianASREventEffect {
guard let data = jsonText.data(using: .utf8),
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
return .none
}
return apply(json: json)
}
mutating func apply(json: [String: Any]) -> BailianASREventEffect {
guard let header = json["header"] as? [String: Any] else { return .none }
let event = header["event"] as? String ?? ""
switch event {
case "task-started":
started = true
return .started
case "result-generated":
return applyResultGenerated(json: json)
case "task-finished":
if finalSegments.isEmpty {
return .finished(lastResultText)
}
let ordered = finalSegments.keys.sorted().compactMap { finalSegments[$0] }
return .finished(BailianRealtimeASRClient.mergeSegments(ordered))
case "task-failed":
let message = header["error_message"] as? String ?? "task failed"
return .failed(message)
default:
return .none
}
}
private mutating func applyResultGenerated(json: [String: Any]) -> BailianASREventEffect {
guard let payload = json["payload"] as? [String: Any],
let output = payload["output"] as? [String: Any],
let sentenceObj = output["sentence"] as? [String: Any] else {
return .none
}
if sentenceObj["heartbeat"] as? Bool == true { return .none }
guard let rawText = sentenceObj["text"] as? String else { return .none }
let trimmed = rawText.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return .none }
lastResultText = trimmed
let sentenceID = sentenceObj["sentence_id"] as? Int64 ?? 0
let sentenceEndValue = sentenceObj["sentence_end"]
let sentenceEnd = sentenceEndValue as? Bool ?? false
let endTime = sentenceObj["end_time"] as? Int64 ?? 0
let isFinal = sentenceEndValue != nil ? sentenceEnd : endTime > 0
if isFinal {
finalSegments[sentenceID] = trimmed
partialSegments.removeValue(forKey: sentenceID)
} else {
partialSegments[sentenceID] = trimmed
}
var displayParts: [String] = []
let ids = Set(finalSegments.keys).union(partialSegments.keys).sorted()
for id in ids {
if let committed = finalSegments[id] {
displayParts.append(committed)
} else if let live = partialSegments[id] {
displayParts.append(live)
}
}
let display = BailianRealtimeASRClient.mergeSegments(displayParts)
.trimmingCharacters(in: .whitespacesAndNewlines)
return display.isEmpty ? .none : .partial(display)
}
}
// MARK: - OpenAI Realtime transcription
enum OpenAIRealtimeEventEffect: Equatable, Sendable {
case none
case sessionReady
case partial(String)
case failed(String)
}
struct OpenAIRealtimeTranscriptReducer: Sendable {
private(set) var sessionReady = false
private(set) var partialByItem: [String: String] = [:]
private(set) var completedByItem: [String: String] = [:]
private(set) var itemOrder: [String] = []
mutating func apply(jsonText: String) -> OpenAIRealtimeEventEffect {
guard let data = jsonText.data(using: .utf8),
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
let type = json["type"] as? String else {
return .none
}
return apply(type: type, json: json)
}
mutating func apply(type: String, json: [String: Any]) -> OpenAIRealtimeEventEffect {
switch type {
case "session.created", "session.updated":
sessionReady = true
return .sessionReady
case "conversation.item.input_audio_transcription.delta":
let itemID = json["item_id"] as? String ?? "default"
let delta = json["delta"] as? String ?? ""
guard !delta.isEmpty else { return .none }
if partialByItem[itemID] == nil, completedByItem[itemID] == nil {
itemOrder.append(itemID)
}
partialByItem[itemID, default: ""] += delta
let display = composedDisplay()
return display.isEmpty ? .none : .partial(display)
case "conversation.item.input_audio_transcription.completed":
let itemID = json["item_id"] as? String ?? "default"
let transcript = (json["transcript"] as? String ?? "")
.trimmingCharacters(in: .whitespacesAndNewlines)
if !itemOrder.contains(itemID) {
itemOrder.append(itemID)
}
if !transcript.isEmpty {
completedByItem[itemID] = transcript
}
partialByItem.removeValue(forKey: itemID)
let display = composedDisplay()
return display.isEmpty ? .none : .partial(display)
case "error":
let message: String
if let error = json["error"] as? [String: Any],
let nested = error["message"] as? String {
message = nested
} else {
message = json["message"] as? String ?? "OpenAI realtime error"
}
return .failed(message)
default:
return .none
}
}
func composedDisplay() -> String {
itemOrder.compactMap { id in
completedByItem[id] ?? partialByItem[id]
}
.joined(separator: " ")
.trimmingCharacters(in: .whitespacesAndNewlines)
}
func composedFinal() -> String {
itemOrder.compactMap { completedByItem[$0] }
.joined(separator: " ")
.trimmingCharacters(in: .whitespacesAndNewlines)
}
static func languageHint(from locale: Locale) -> String? {
let id = locale.identifier.lowercased()
if id.hasPrefix("zh") { return "zh" }
if id.hasPrefix("en") { return "en" }
if id.hasPrefix("ja") { return "ja" }
if id.hasPrefix("ko") { return "ko" }
return nil
}
}
@@ -0,0 +1,344 @@
// OpenAIRealtimeASRClient.swift
// OSGKeyboard · Shared
//
// OpenAI Realtime transcription (WebSocket). Streams PCM and transcript
// deltas for utterance-level ASR. Batch `/audio/transcriptions` remains the
// fallback path when realtime is unavailable.
import Foundation
import os
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
struct OpenAIRealtimeASRClient: CloudASRTranscribing, CloudASRStreamingCapable {
let apiKey: String
let endpoint: String
let model: String
let session: URLSession
/// Used when streaming fails and Flow falls back to chunked batch ASR.
private let batchClient: PromptCloudASRClient
static let appendChunkBytes = 4_800 // 100 ms @ 24 kHz / 16-bit mono.
static let finalTimeout: TimeInterval = 15
init(
apiKey: String,
endpoint: String,
model: String,
batchBaseURL: String,
session: URLSession
) {
self.apiKey = apiKey
self.endpoint = endpoint
self.model = model
self.session = session
self.batchClient = PromptCloudASRClient(
providerId: "openai",
baseURL: batchBaseURL.isEmpty ? "https://api.openai.com/v1" : batchBaseURL,
apiKey: apiKey,
model: Self.batchModel(from: model),
session: session
)
}
func prepare(dictionary: PersonalDictionary) async throws {}
func openStreamingSession(
locale: Locale,
dictionary: PersonalDictionary,
onPartial: @escaping @Sendable (String) -> Void
) async throws -> any CloudASRStreamingSession {
_ = dictionary
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
let url = try resolvedEndpointURL()
var request = URLRequest(url: url)
request.timeoutInterval = 8
request.setValue(
"Bearer \(apiKey.trimmingCharacters(in: .whitespacesAndNewlines))",
forHTTPHeaderField: "Authorization"
)
let wsTask = session.webSocketTask(with: request)
wsTask.resume()
let live = OpenAIRealtimeStreamingSession(
wsTask: wsTask,
model: resolvedRealtimeModel,
locale: locale,
onPartial: onPartial
)
try await live.start()
return live
}
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String {
try await batchClient.transcribe(
samples: samples,
sampleRate: sampleRate,
locale: locale,
dictionary: dictionary
)
}
func probeConnection() async throws {
do {
let session = try await openStreamingSession(
locale: Locale(identifier: "zh-CN"),
dictionary: .empty,
onPartial: { _ in }
)
session.cancel()
} catch {
try await batchClient.probeConnection()
}
}
private var resolvedRealtimeModel: String {
let trimmed = model.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty || trimmed.hasPrefix("gpt-4o") || trimmed == "whisper-1" {
return CloudASRModelCatalog.openAIRealtimeWhisper
}
return trimmed
}
private func resolvedEndpointURL() throws -> URL {
let raw = endpoint.trimmingCharacters(in: .whitespacesAndNewlines)
if raw.hasPrefix("wss://") || raw.hasPrefix("ws://") {
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
return url
}
guard let url = URL(string: CloudASRModelCatalog.openAIRealtimeEndpoint) else {
throw CloudASRError.invalidURL
}
return url
}
private static func batchModel(from model: String) -> String {
let trimmed = model.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty || trimmed.contains("realtime") {
return CloudASRModelCatalog.openAITranscribe
}
return trimmed
}
}
// MARK: - Utterance session
private final class OpenAIRealtimeStreamingSession: CloudASRStreamingSession, @unchecked Sendable {
private let wsTask: URLSessionWebSocketTask
private let model: String
private let locale: Locale
private let onPartial: @Sendable (String) -> Void
private let lock = OSAllocatedUnfairLock()
private var receiveTask: Task<Void, Never>?
private var failure: Error?
private var finished = false
private var pcmBuffer = Data()
private var reducer = OpenAIRealtimeTranscriptReducer()
private var awaitingCommit = false
init(
wsTask: URLSessionWebSocketTask,
model: String,
locale: Locale,
onPartial: @escaping @Sendable (String) -> Void
) {
self.wsTask = wsTask
self.model = model
self.locale = locale
self.onPartial = onPartial
}
func start() async throws {
receiveTask = Task { [weak self] in
await self?.receiveLoop()
}
let language = OpenAIRealtimeTranscriptReducer.languageHint(from: locale)
var transcription: [String: Any] = [
"model": model,
"delay": "low",
]
if let language {
transcription["language"] = language
}
var input: [String: Any] = [
"format": [
"type": "audio/pcm",
"rate": 24_000,
],
"transcription": transcription,
]
input["turn_detection"] = NSNull()
let update: [String: Any] = [
"type": "session.update",
"session": [
"type": "transcription",
"audio": [
"input": input,
],
],
]
try await sendJSON(update)
let deadline = Date().addingTimeInterval(8)
while Date() < deadline {
try throwIfFailed()
if lock.withLock({ reducer.sessionReady }) { return }
try await Task.sleep(nanoseconds: 20_000_000)
}
cancel()
throw CloudASRError.transport("OpenAI realtime session timed out")
}
func append(samples: [Float]) async throws {
try throwIfFailed()
let upsampled = CloudASRStreamingPCM.upsample16kTo24k(samples)
let pcm = CloudASRStreamingPCM.pcm16LE(samples: upsampled)
let frames: [Data] = lock.withLock {
pcmBuffer.append(pcm)
var frames: [Data] = []
while pcmBuffer.count >= OpenAIRealtimeASRClient.appendChunkBytes {
let frame = pcmBuffer.prefix(OpenAIRealtimeASRClient.appendChunkBytes)
frames.append(Data(frame))
pcmBuffer.removeFirst(OpenAIRealtimeASRClient.appendChunkBytes)
}
return frames
}
for frame in frames {
try await sendAppend(frame)
}
}
func finish() async throws -> String {
try throwIfFailed()
let trailing: Data = lock.withLock {
let data = pcmBuffer
pcmBuffer.removeAll(keepingCapacity: false)
awaitingCommit = true
return data
}
if !trailing.isEmpty {
try await sendAppend(trailing)
}
try await sendJSON(["type": "input_audio_buffer.commit"])
let deadline = Date().addingTimeInterval(OpenAIRealtimeASRClient.finalTimeout)
while Date() < deadline {
try throwIfFailed()
let snapshot = lock.withLock {
(awaitingCommit, reducer.composedFinal(), reducer.composedDisplay())
}
if !snapshot.0 {
let text = snapshot.1.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
? snapshot.2
: snapshot.1
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
cancel()
if trimmed.isEmpty { throw CloudASRError.emptyTranscript }
return trimmed
}
let settled = lock.withLock {
!reducer.completedByItem.isEmpty && reducer.partialByItem.isEmpty && !awaitingCommit
}
if settled {
let text = lock.withLock { reducer.composedFinal() }
.trimmingCharacters(in: .whitespacesAndNewlines)
cancel()
if text.isEmpty { throw CloudASRError.emptyTranscript }
return text
}
try await Task.sleep(nanoseconds: 20_000_000)
}
let fallback = lock.withLock {
let final = reducer.composedFinal()
return final.isEmpty ? reducer.composedDisplay() : final
}
.trimmingCharacters(in: .whitespacesAndNewlines)
cancel()
if fallback.isEmpty {
throw CloudASRError.transport("OpenAI realtime final timed out")
}
return fallback
}
func cancel() {
receiveTask?.cancel()
wsTask.cancel(with: .normalClosure, reason: nil)
lock.withLock { finished = true }
}
private func receiveLoop() async {
while !Task.isCancelled {
let message: URLSessionWebSocketTask.Message
do {
message = try await wsTask.receive()
} catch {
publishFailure(CloudASRError.transport(error.localizedDescription))
return
}
let text: String
switch message {
case .string(let value):
text = value
case .data(let data):
text = String(data: data, encoding: .utf8) ?? ""
@unknown default:
continue
}
guard !text.isEmpty else { continue }
let effect = lock.withLock { () -> OpenAIRealtimeEventEffect in
let effect = reducer.apply(jsonText: text)
if case .partial = effect, !reducer.completedByItem.isEmpty {
awaitingCommit = false
}
return effect
}
switch effect {
case .none, .sessionReady:
continue
case .partial(let display):
onPartial(display)
case .failed(let message):
publishFailure(CloudASRError.transport(message))
return
}
}
}
private func sendAppend(_ pcm: Data) async throws {
let audio = pcm.base64EncodedString()
try await sendJSON([
"type": "input_audio_buffer.append",
"audio": audio,
])
}
private func sendJSON(_ body: [String: Any]) async throws {
guard JSONSerialization.isValidJSONObject(body),
let data = try? JSONSerialization.data(withJSONObject: body),
let string = String(data: data, encoding: .utf8) else {
throw CloudASRError.decoding("invalid realtime payload")
}
do {
try await wsTask.send(.string(string))
} catch {
throw CloudASRError.transport(error.localizedDescription)
}
}
private func throwIfFailed() throws {
let (error, done) = lock.withLock { (failure, finished) }
if let error { throw error }
if done { throw CloudASRError.transport("OpenAI realtime session cancelled") }
}
private func publishFailure(_ error: Error) {
lock.withLock { failure = error }
cancel()
}
}
@@ -0,0 +1,578 @@
// VolcengineCloudASRClient.swift
// OSGKeyboard · Shared
//
// Volcengine SAUC bigmodel ASR client. Utterance-level WebSocket session with
// enable_nonstream (official two-pass): interim text for on-screen partials,
// definite utterances for polish-ready finals.
import Foundation
import os
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
struct VolcengineCloudASRClient: CloudASRTranscribing, CloudASRStreamingCapable {
let apiKey: String
let endpoint: String
let resourceID: String
let session: URLSession
static let targetChunkBytes = 6_400 // 200 ms @ 16 kHz, 16-bit, mono.
static let finalTimeout: TimeInterval = 12
private static let hotwordCap = 80
func prepare(dictionary: PersonalDictionary) async throws {}
func openStreamingSession(
locale: Locale,
dictionary: PersonalDictionary,
onPartial: @escaping @Sendable (String) -> Void
) async throws -> any CloudASRStreamingSession {
_ = locale
let credentials = try VolcengineCredentials.parse(
apiKey: apiKey,
fallbackResourceID: resolvedResourceID
)
let url = try resolvedEndpointURL()
let connectID = UUID().uuidString
var request = URLRequest(url: url)
request.timeoutInterval = 8
request.setValue(credentials.appID, forHTTPHeaderField: "X-Api-App-Key")
request.setValue(credentials.accessToken, forHTTPHeaderField: "X-Api-Access-Key")
request.setValue(credentials.resourceID, forHTTPHeaderField: "X-Api-Resource-Id")
request.setValue(connectID, forHTTPHeaderField: "X-Api-Connect-Id")
let task = session.webSocketTask(with: request)
task.resume()
let live = VolcengineStreamingSession(
wsTask: task,
connectID: connectID,
dictionary: dictionary,
onPartial: onPartial
)
try await live.start()
return live
}
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String {
_ = sampleRate
guard !samples.isEmpty else { throw CloudASRError.emptyTranscript }
let session = try await openStreamingSession(
locale: locale,
dictionary: dictionary,
onPartial: { _ in }
)
try await session.append(samples: samples)
let text = try await session.finish()
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { throw CloudASRError.emptyTranscript }
return trimmed
}
/// Settings connection probe: WebSocket upgrade + first-frame auth only.
///
/// Do not push silence through `transcribe` SAUC returns an empty final
/// then closes the socket, and `finish()` prefers that Socket error over
/// `emptyTranscript`, so the default silence probe always failed while
/// real dictation (with speech) still worked.
func probeConnection() async throws {
let live = try await openStreamingSession(
locale: Locale(identifier: "zh-CN"),
dictionary: .empty,
onPartial: { _ in }
)
live.cancel()
}
private var resolvedResourceID: String {
resourceID.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
? CloudASRModelCatalog.volcengineDefaultResourceID
: resourceID.trimmingCharacters(in: .whitespacesAndNewlines)
}
private func resolvedEndpointURL() throws -> URL {
let raw = endpoint.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
? CloudASRModelCatalog.volcengineEndpoint
: endpoint.trimmingCharacters(in: .whitespacesAndNewlines)
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
return url
}
static func firstFramePayload(
connectID: String,
dictionary: PersonalDictionary
) throws -> Data {
var request: [String: Any] = [
"model_name": "bigmodel",
"enable_itn": true,
"enable_punc": true,
"show_utterances": true,
"enable_speaker_info": true,
// Official two-pass: stream interim for UI, nostream re-decode per
// VAD sentence for definite polish-ready text (scheme A).
"enable_nonstream": true,
"end_window_size": 800,
"force_to_speech_time": 1_000,
]
if let context = hotwordContext(dictionary: dictionary) {
request["context"] = context
}
let payload: [String: Any] = [
"user": ["uid": connectID],
"audio": [
"format": "pcm",
"rate": 16_000,
"bits": 16,
"channel": 1,
"codec": "raw",
],
"request": request,
]
return try JSONSerialization.data(withJSONObject: payload)
}
private static func hotwordContext(dictionary: PersonalDictionary) -> String? {
var seen: [String] = []
for word in dictionary.asrHotwords() {
let trimmed = word.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { continue }
guard !seen.contains(where: { $0.caseInsensitiveCompare(trimmed) == .orderedSame }) else {
continue
}
seen.append(trimmed)
if seen.count >= hotwordCap { break }
}
guard !seen.isEmpty else { return nil }
let words = seen.map { ["word": $0] }
guard let data = try? JSONSerialization.data(withJSONObject: ["hotwords": words]) else {
return nil
}
return String(data: data, encoding: .utf8)
}
static func displayText(from payload: Data) -> String {
guard let json = try? JSONSerialization.jsonObject(with: payload) as? [String: Any],
let result = normalizedResult(from: json) else {
return ""
}
if let utterances = result["utterances"] as? [[String: Any]], !utterances.isEmpty {
let pieces = utterances.compactMap { $0["text"] as? String }
let joined = pieces.joined()
if !joined.isEmpty { return joined }
}
return result["text"] as? String ?? ""
}
/// Prefer definite (two-pass) utterance text for polish input.
static func committedText(from payload: Data) -> String {
guard let json = try? JSONSerialization.jsonObject(with: payload) as? [String: Any],
let result = normalizedResult(from: json),
let utterances = result["utterances"] as? [[String: Any]],
!utterances.isEmpty else {
return ""
}
let definite = utterances.compactMap { utterance -> String? in
let isDefinite = utterance["definite"] as? Bool ?? false
guard isDefinite else { return nil }
return utterance["text"] as? String
}
return definite.joined()
}
private static func normalizedResult(from json: [String: Any]) -> [String: Any]? {
if let result = json["result"] as? [String: Any] {
return result
}
if let results = json["result"] as? [[String: Any]] {
return results.first
}
if json["text"] as? String != nil {
return json
}
return nil
}
}
// MARK: - Utterance session
private final class VolcengineStreamingSession: CloudASRStreamingSession, @unchecked Sendable {
private let wsTask: URLSessionWebSocketTask
private let connectID: String
private let dictionary: PersonalDictionary
private let onPartial: @Sendable (String) -> Void
private let lock = OSAllocatedUnfairLock()
private var sequence: Int32 = 1
private var pcmBuffer = Data()
private var receiveTask: Task<Void, Never>?
private var failure: Error?
private var finished = false
private var lastDisplay = ""
private var lastCommitted = ""
private var sawServerFinal = false
init(
wsTask: URLSessionWebSocketTask,
connectID: String,
dictionary: PersonalDictionary,
onPartial: @escaping @Sendable (String) -> Void
) {
self.wsTask = wsTask
self.connectID = connectID
self.dictionary = dictionary
self.onPartial = onPartial
}
func start() async throws {
let firstPayload = try VolcengineCloudASRClient.firstFramePayload(
connectID: connectID,
dictionary: dictionary
)
try await send(
VolcengineFrame.build(
messageType: .fullClientRequest,
flags: .positiveSequence,
serialization: .json,
payload: firstPayload,
sequence: 1
)
)
sequence = 2
receiveTask = Task { [weak self] in
await self?.receiveLoop()
}
}
func append(samples: [Float]) async throws {
try throwIfFailed()
let pcm = CloudASRStreamingPCM.pcm16LE(samples: samples)
let (frames, nextSequences): ([Data], [Int32]) = lock.withLock {
pcmBuffer.append(pcm)
var frames: [Data] = []
while pcmBuffer.count >= VolcengineCloudASRClient.targetChunkBytes {
let frame = pcmBuffer.prefix(VolcengineCloudASRClient.targetChunkBytes)
frames.append(Data(frame))
pcmBuffer.removeFirst(VolcengineCloudASRClient.targetChunkBytes)
}
let nextSequences: [Int32] = frames.indices.map { _ in
let seq = sequence
sequence += 1
return seq
}
return (frames, nextSequences)
}
for (frame, seq) in zip(frames, nextSequences) {
try await send(
VolcengineFrame.build(
messageType: .audioOnlyRequest,
flags: .positiveSequence,
serialization: .none,
payload: frame,
sequence: seq
)
)
}
}
func finish() async throws -> String {
try throwIfFailed()
// Only consume a sequence number when we actually send trailing PCM.
// Skipping an unused seq (common when length is an exact chunk multiple,
// e.g. the settings probe's 1 s / 32_000-byte clip) makes the final
// negative packet mismatch server autoAssignedSequence error 45000000.
let trailing = lock.withLock { () -> Data in
let data = pcmBuffer
pcmBuffer.removeAll(keepingCapacity: false)
return data
}
if !trailing.isEmpty {
let endSequence = lock.withLock { () -> Int32 in
let seq = sequence
sequence += 1
return seq
}
try await send(
VolcengineFrame.build(
messageType: .audioOnlyRequest,
flags: .positiveSequence,
serialization: .none,
payload: trailing,
sequence: endSequence
)
)
}
let negativeSeq = lock.withLock { () -> Int32 in
let seq = sequence
sequence += 1
return seq
}
try await send(
VolcengineFrame.build(
messageType: .audioOnlyRequest,
flags: .negativeSequence,
serialization: .none,
payload: Data(),
sequence: -negativeSeq
)
)
let deadline = Date().addingTimeInterval(VolcengineCloudASRClient.finalTimeout)
while Date() < deadline {
try throwIfFailed()
let snapshot = lock.withLock { (sawServerFinal, lastCommitted, lastDisplay) }
if snapshot.0 {
let text = snapshot.1.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
? snapshot.2
: snapshot.1
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
cancel()
if trimmed.isEmpty { throw CloudASRError.emptyTranscript }
return trimmed
}
try await Task.sleep(nanoseconds: 20_000_000)
}
cancel()
throw CloudASRError.transport("Volcengine final result timed out")
}
func cancel() {
receiveTask?.cancel()
wsTask.cancel(with: .normalClosure, reason: nil)
lock.withLock { finished = true }
}
private func receiveLoop() async {
while !Task.isCancelled {
let message: URLSessionWebSocketTask.Message
do {
message = try await wsTask.receive()
} catch {
publishFailure(CloudASRError.transport(error.localizedDescription))
return
}
let data: Data
switch message {
case .data(let payload):
data = payload
case .string(let string):
data = Data(string.utf8)
@unknown default:
continue
}
guard let frame = VolcengineFrame.parse(data) else { continue }
if frame.messageType == .errorMessage {
let body = String(data: frame.payload, encoding: .utf8) ?? ""
let code = frame.errorCode ?? 0
publishFailure(CloudASRError.transport("ASR error \(code): \(body)"))
return
}
guard frame.messageType == .fullServerResponse else { continue }
let display = VolcengineCloudASRClient.displayText(from: frame.payload)
.trimmingCharacters(in: .whitespacesAndNewlines)
let committed = VolcengineCloudASRClient.committedText(from: frame.payload)
.trimmingCharacters(in: .whitespacesAndNewlines)
let emit = lock.withLock { () -> String in
if !display.isEmpty {
lastDisplay = display
}
if !committed.isEmpty {
lastCommitted = committed
}
if frame.isFinal {
sawServerFinal = true
}
return lastDisplay
}
if !emit.isEmpty {
onPartial(emit)
}
}
}
private func send(_ data: Data) async throws {
do {
try await wsTask.send(.data(data))
} catch {
throw CloudASRError.transport(error.localizedDescription)
}
}
private func throwIfFailed() throws {
let (error, done) = lock.withLock { (failure, finished) }
if let error { throw error }
if done { throw CloudASRError.transport("Volcengine session cancelled") }
}
private func publishFailure(_ error: Error) {
lock.withLock { failure = error }
cancel()
}
}
private struct VolcengineCredentials {
let appID: String
let accessToken: String
let resourceID: String
static func parse(apiKey: String, fallbackResourceID: String) throws -> VolcengineCredentials {
let trimmed = apiKey.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { throw CloudASRError.noAPIKey }
if let data = trimmed.data(using: .utf8),
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {
let appID = string(json, keys: ["app_id", "appId", "appid"])
let token = string(json, keys: ["access_token", "accessToken", "token"])
let resourceID = string(json, keys: ["resource_id", "resourceId", "resource"])
?? fallbackResourceID
guard let appID, let token, !resourceID.isEmpty else { throw CloudASRError.noAPIKey }
return VolcengineCredentials(appID: appID, accessToken: token, resourceID: resourceID)
}
let separators = CharacterSet(charactersIn: ":\n,")
let parts = trimmed
.components(separatedBy: separators)
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
.filter { !$0.isEmpty }
guard parts.count >= 2 else { throw CloudASRError.noAPIKey }
let resourceID = parts.count >= 3 ? parts[2] : fallbackResourceID
return VolcengineCredentials(appID: parts[0], accessToken: parts[1], resourceID: resourceID)
}
private static func string(_ json: [String: Any], keys: [String]) -> String? {
for key in keys {
if let value = json[key] as? String {
let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty { return trimmed }
}
}
return nil
}
}
enum VolcengineMessageType: UInt8 {
case fullClientRequest = 0b0001
case audioOnlyRequest = 0b0010
case fullServerResponse = 0b1001
case errorMessage = 0b1111
}
enum VolcengineFlags: UInt8 {
case none = 0b0000
case positiveSequence = 0b0001
case lastPacket = 0b0010
case negativeSequence = 0b0011
}
enum VolcengineSerialization: UInt8 {
case none = 0b0000
case json = 0b0001
}
struct VolcengineFrame {
let messageType: VolcengineMessageType?
let flags: UInt8
let sequence: Int32?
let errorCode: UInt32?
let payload: Data
var isFinal: Bool {
flags == VolcengineFlags.lastPacket.rawValue
|| flags == VolcengineFlags.negativeSequence.rawValue
|| (sequence ?? 0) < 0
}
static func build(
messageType: VolcengineMessageType,
flags: VolcengineFlags,
serialization: VolcengineSerialization,
payload: Data,
sequence: Int32?
) -> Data {
var data = Data()
data.append(0x11)
data.append((messageType.rawValue << 4) | flags.rawValue)
data.append(serialization.rawValue << 4)
data.append(0x00)
if flags == .positiveSequence || flags == .negativeSequence, let sequence {
data.appendBE32(UInt32(bitPattern: sequence))
}
data.appendBE32(UInt32(payload.count))
data.append(payload)
return data
}
static func parse(_ data: Data) -> VolcengineFrame? {
guard data.count >= 8 else { return nil }
let bytes = [UInt8](data)
let headerSize = Int(bytes[0] & 0x0F) * 4
guard headerSize >= 4, data.count >= headerSize + 4 else { return nil }
let typeRaw = (bytes[1] >> 4) & 0x0F
let messageType = VolcengineMessageType(rawValue: typeRaw)
let flags = bytes[1] & 0x0F
let compression = bytes[2] & 0x0F
guard compression == 0 else { return nil }
var offset = headerSize
var sequence: Int32?
if flags == VolcengineFlags.positiveSequence.rawValue
|| flags == VolcengineFlags.negativeSequence.rawValue {
guard let value = data.readBE32(at: offset) else { return nil }
sequence = Int32(bitPattern: value)
offset += 4
}
if messageType == .errorMessage {
guard let code = data.readBE32(at: offset),
let size = data.readBE32(at: offset + 4) else { return nil }
offset += 8
guard data.count >= offset + Int(size) else { return nil }
return VolcengineFrame(
messageType: messageType,
flags: flags,
sequence: sequence,
errorCode: code,
payload: data.subdata(in: offset..<(offset + Int(size)))
)
}
guard let size = data.readBE32(at: offset) else { return nil }
offset += 4
guard data.count >= offset + Int(size) else { return nil }
return VolcengineFrame(
messageType: messageType,
flags: flags,
sequence: sequence,
errorCode: nil,
payload: data.subdata(in: offset..<(offset + Int(size)))
)
}
}
private extension Data {
mutating func appendBE32(_ value: UInt32) {
var bigEndian = value.bigEndian
Swift.withUnsafeBytes(of: &bigEndian) { append(contentsOf: $0) }
}
func readBE32(at offset: Int) -> UInt32? {
guard count >= offset + 4 else { return nil }
return self[offset..<(offset + 4)].reduce(UInt32(0)) { ($0 << 8) | UInt32($1) }
}
}
@@ -0,0 +1,449 @@
// CustomLanguageModelManager.swift
// OSGKeyboard · Shared
//
// Prepares the bundled SFCustomLanguageModelData asset on device and shares
// the compiled LM + Vocab through the App Group container. Both the host app
// and keyboard extension read the same prepared configuration for
// DictationTranscriber content hints.
import Foundation
import Speech
import os
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
public final class CustomLanguageModelManager: @unchecked Sendable {
public static let shared = CustomLanguageModelManager()
public enum PrepareState: Equatable, Sendable {
case idle
case preparing
case ready
case failed(String)
}
struct BundledManifest: Decodable {
let version: String
let bin_bytes: Int
let identifier: String
}
private enum Storage {
static let subdirectory = "CustomLanguageModel/v1"
static let fingerprintKey = "customLM.preparedFingerprint"
static let preparedAtKey = "customLM.preparedAt"
static let lastFailureAtKey = "customLM.lastFailureAt"
static let attemptCountKey = "customLM.attemptCount"
static let maxRetryAttempts = 3
/// Backoff after failure attempts 1, 2, and 3 (seconds).
static let backoffIntervals: [TimeInterval] = [30, 120, 600]
}
private let lock = OSAllocatedUnfairLock()
private var cachedConfiguration: SFSpeechLanguageModel.Configuration?
private var state: PrepareState = .idle
private var prepareTask: Task<Void, Never>?
private init() {}
// MARK: - Public API
/// Returns a prepared configuration for Chinese locales when available.
public func configurationForTranscription(locale: Locale) -> SFSpeechLanguageModel.Configuration? {
guard Self.isChineseLocale(locale) else { return nil }
return lock.withLock { () -> SFSpeechLanguageModel.Configuration? in
if let cachedConfiguration {
return cachedConfiguration
}
if let loaded = Self.loadCachedConfigurationFromDisk() {
cachedConfiguration = loaded
state = .ready
return loaded
}
return nil
}
}
public func currentState() -> PrepareState {
lock.withLock { state }
}
/// Fire-and-forget preparation for the host app. Safe to call repeatedly.
/// Retries after exponential backoff when a prior attempt failed.
public func prepareInBackgroundIfNeeded() {
#if os(iOS)
guard AppGroup.isAvailable else { return }
#endif
let shouldStart = lock.withLock { () -> Bool in
if case .preparing = state { return false }
if cachedConfiguration != nil { return false }
if let loaded = Self.loadCachedConfigurationFromDisk() {
cachedConfiguration = loaded
state = .ready
Self.clearRetryState()
return false
}
if prepareTask != nil { return false }
if case .failed = state {
guard Self.canRetryAfterFailure() else { return false }
} else if !Self.canRetryAfterFailure() {
return false
}
state = .preparing
return true
}
guard shouldStart else { return }
prepareTask = Task.detached(priority: .utility) { [weak self] in
guard let self else { return }
defer {
self.lock.withLock { self.prepareTask = nil }
}
do {
_ = try await self.prepareIfNeeded()
} catch {
Self.recordFailure()
self.lock.withLock {
self.state = .failed(error.localizedDescription)
}
Self.log(
"prepare failed (attempt \(Self.storedAttemptCount())): \(error.localizedDescription)"
)
}
}
}
/// Prepares the bundled training asset into the App Group container.
@discardableResult
public func prepareIfNeeded() async throws -> SFSpeechLanguageModel.Configuration? {
if let existing = configurationForTranscription(locale: Locale(identifier: "zh-Hans")) {
lock.withLock { state = .ready }
Self.clearRetryState()
return existing
}
guard Self.canRetryAfterFailure() else {
throw PrepareError.retryBudgetExhausted
}
guard let manifest = Self.bundledManifest() else {
throw PrepareError.missingManifest
}
guard let assetURL = Self.bundledTrainingAssetURL() else {
throw PrepareError.missingTrainingAsset
}
guard let preparedDir = Self.preparedDirectoryURL() else {
throw PrepareError.missingAppGroupContainer
}
let fingerprint = Self.fingerprint(for: manifest)
if Self.storedFingerprint() == fingerprint,
let cached = Self.loadCachedConfigurationFromDisk() {
lock.withLock {
cachedConfiguration = cached
state = .ready
}
Self.clearRetryState()
return cached
}
lock.withLock { state = .preparing }
let languageModelURL = preparedDir.appendingPathComponent("LM")
let vocabularyURL = preparedDir.appendingPathComponent("Vocab")
try Self.removeItemIfExists(at: languageModelURL)
try Self.removeItemIfExists(at: vocabularyURL)
let configuration = SFSpeechLanguageModel.Configuration(
languageModel: languageModelURL,
vocabulary: vocabularyURL
)
Self.log(
"preparing custom LM (\(manifest.bin_bytes) byte asset)… \(OSGDiag.memoryTag())"
)
OSGDiag.log(
"clm.prepare.begin bytes=\(manifest.bin_bytes) \(OSGDiag.memoryTag())",
category: "asr"
)
try await Self.prepareLanguageModel(assetURL: assetURL, configuration: configuration)
guard FileManager.default.fileExists(atPath: languageModelURL.path),
FileManager.default.fileExists(atPath: vocabularyURL.path) else {
throw PrepareError.missingPreparedArtifacts
}
Self.persistenceDefaults.set(fingerprint, forKey: Storage.fingerprintKey)
Self.persistenceDefaults.set(Date().timeIntervalSince1970, forKey: Storage.preparedAtKey)
Self.clearRetryState()
lock.withLock {
cachedConfiguration = configuration
state = .ready
}
Self.log("custom LM ready at \(preparedDir.path)")
OSGDiag.log(
"clm.prepare.done path=\(preparedDir.lastPathComponent) \(OSGDiag.memoryTag())",
category: "asr"
)
return configuration
}
// MARK: - DictationTranscriber factory (iOS host app)
#if os(iOS)
public static func makeDictationTranscriber(
locale: Locale,
lmConfiguration: SFSpeechLanguageModel.Configuration?
) -> DictationTranscriber {
let preset = DictationTranscriber.Preset.progressiveLongDictation
guard let lmConfiguration, isChineseLocale(locale) else {
return DictationTranscriber(locale: locale, preset: preset)
}
let contentHints = preset.contentHints.union([
.customizedLanguage(modelConfiguration: lmConfiguration),
])
return DictationTranscriber(
locale: locale,
contentHints: contentHints,
transcriptionOptions: preset.transcriptionOptions,
reportingOptions: preset.reportingOptions,
attributeOptions: preset.attributeOptions
)
}
#endif
// MARK: - Legacy Speech request (macOS Apple Speech fallback)
/// Up to 100 short phrases for `SFSpeechRecognitionRequest.contextualStrings`.
public static func contextualStringsForRecognition(
bias: LocalASRBiasPayload?,
maxCount: Int = 100
) -> [String] {
guard let bias, !bias.hardHotwords.isEmpty else { return [] }
return Array(bias.hardHotwords.prefix(max(1, maxCount)))
}
/// Applies bundled CLM + optional contextual strings to a legacy on-device request.
public static func applyCustomLanguageModel(
to request: SFSpeechURLRecognitionRequest,
locale: Locale,
bias: LocalASRBiasPayload?
) {
request.requiresOnDeviceRecognition = true
if let configuration = shared.configurationForTranscription(locale: locale) {
request.customizedLanguageModel = configuration
}
let phrases = contextualStringsForRecognition(bias: bias)
if !phrases.isEmpty {
request.contextualStrings = phrases
}
}
// MARK: - Bundle / disk helpers
private static var resourceBundle: Bundle {
// CLM assets ship in the host app bundle (not the extension Shared
// framework) so the keyboard process never mmaps the training bin.
Bundle.main
}
static func bundledTrainingAssetURL() -> URL? {
// Prefer flat Bundle.main (iOS HostCLM copy), then legacy subdirs.
let candidates: [URL?] = [
resourceBundle.url(forResource: "OSGKeyboardCLM", withExtension: "bin"),
resourceBundle.url(
forResource: "OSGKeyboardCLM",
withExtension: "bin",
subdirectory: "HostCLM/v1"
),
resourceBundle.url(
forResource: "OSGKeyboardCLM",
withExtension: "bin",
subdirectory: Storage.subdirectory
),
]
return candidates.compactMap { $0 }.first
}
static func bundledManifest() -> BundledManifest? {
let candidates: [URL?] = [
resourceBundle.url(forResource: "compiled-manifest", withExtension: "json"),
resourceBundle.url(
forResource: "compiled-manifest",
withExtension: "json",
subdirectory: "HostCLM/v1"
),
resourceBundle.url(
forResource: "compiled-manifest",
withExtension: "json",
subdirectory: Storage.subdirectory
),
]
guard let manifestURL = candidates.compactMap({ $0 }).first,
let data = try? Data(contentsOf: manifestURL),
let manifest = try? JSONDecoder().decode(BundledManifest.self, from: data)
else {
return nil
}
return manifest
}
static func preparedDirectoryURL() -> URL? {
if let container = FileManager.default.containerURL(
forSecurityApplicationGroupIdentifier: AppGroup.identifier
) {
let directory = container.appendingPathComponent(Storage.subdirectory, isDirectory: true)
try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
return directory
}
#if os(macOS)
guard let appSupport = FileManager.default.urls(
for: .applicationSupportDirectory,
in: .userDomainMask
).first else {
return nil
}
let directory = appSupport
.appendingPathComponent("OSGKeyboard", isDirectory: true)
.appendingPathComponent(Storage.subdirectory, isDirectory: true)
try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
return directory
#else
return nil
#endif
}
static func loadCachedConfigurationFromDisk() -> SFSpeechLanguageModel.Configuration? {
guard let manifest = bundledManifest(),
storedFingerprint() == fingerprint(for: manifest),
let preparedDir = preparedDirectoryURL()
else {
return nil
}
let languageModelURL = preparedDir.appendingPathComponent("LM")
let vocabularyURL = preparedDir.appendingPathComponent("Vocab")
let fm = FileManager.default
guard fm.fileExists(atPath: languageModelURL.path),
fm.fileExists(atPath: vocabularyURL.path) else {
return nil
}
return SFSpeechLanguageModel.Configuration(
languageModel: languageModelURL,
vocabulary: vocabularyURL
)
}
static func isChineseLocale(_ locale: Locale) -> Bool {
locale.identifier(.bcp47).lowercased().hasPrefix("zh")
}
private static func fingerprint(for manifest: BundledManifest) -> String {
"\(manifest.identifier)|\(manifest.version)|\(manifest.bin_bytes)"
}
private static func storedFingerprint() -> String? {
persistenceDefaults.string(forKey: Storage.fingerprintKey)
}
private static func removeItemIfExists(at url: URL) throws {
let fm = FileManager.default
if fm.fileExists(atPath: url.path) {
try fm.removeItem(at: url)
}
}
private static func prepareLanguageModel(
assetURL: URL,
configuration: SFSpeechLanguageModel.Configuration
) async throws {
try await withCheckedThrowingContinuation {
(continuation: CheckedContinuation<Void, Error>) in
SFSpeechLanguageModel.prepareCustomLanguageModel(
for: assetURL,
configuration: configuration
) { error in
if let error {
continuation.resume(throwing: error)
} else {
continuation.resume()
}
}
}
}
// MARK: - Retry / backoff
private static var persistenceDefaults: UserDefaults {
AppGroup.defaultsIfAvailable ?? .standard
}
private static func storedAttemptCount() -> Int {
persistenceDefaults.integer(forKey: Storage.attemptCountKey)
}
private static func storedLastFailureAt() -> TimeInterval? {
let value = persistenceDefaults.double(forKey: Storage.lastFailureAtKey)
return value > 0 ? value : nil
}
private static func recordFailure() {
let nextAttempt = storedAttemptCount() + 1
persistenceDefaults.set(nextAttempt, forKey: Storage.attemptCountKey)
persistenceDefaults.set(Date().timeIntervalSince1970, forKey: Storage.lastFailureAtKey)
}
private static func clearRetryState() {
persistenceDefaults.removeObject(forKey: Storage.attemptCountKey)
persistenceDefaults.removeObject(forKey: Storage.lastFailureAtKey)
}
/// Returns false when retry budget is exhausted or backoff has not elapsed.
private static func canRetryAfterFailure() -> Bool {
let attempts = storedAttemptCount()
guard attempts > 0 else { return true }
guard attempts <= Storage.maxRetryAttempts else { return false }
guard let lastFailureAt = storedLastFailureAt() else { return true }
let backoffIndex = min(attempts - 1, Storage.backoffIntervals.count - 1)
let requiredDelay = Storage.backoffIntervals[backoffIndex]
let elapsed = Date().timeIntervalSince1970 - lastFailureAt
return elapsed >= requiredDelay
}
private static func log(_ message: String) {
OSGLog.clm.info("\(message, privacy: .public)")
}
enum PrepareError: LocalizedError {
case missingManifest
case missingTrainingAsset
case missingAppGroupContainer
case missingPreparedArtifacts
case retryBudgetExhausted
var errorDescription: String? {
switch self {
case .missingManifest:
return "Missing bundled custom language model manifest."
case .missingTrainingAsset:
return "Missing bundled custom language model training asset."
case .missingAppGroupContainer:
return "App Group container unavailable for custom language model preparation."
case .missingPreparedArtifacts:
return "Custom language model preparation did not produce LM/Vocab artifacts."
case .retryBudgetExhausted:
return "Custom language model preparation retry budget exhausted."
}
}
}
}
@@ -0,0 +1,245 @@
// FlowAudioSessionCoordinator.swift
// OSGKeyboard · Host Support
//
// Single owner for the shared iOS AVAudioSession used by Flow capture and
// Picture in Picture keep-alive. Keeping category transitions here prevents
// two independent state machines from racing the same process-wide session.
import AVFoundation
import Foundation
import OSGKeyboardShared
public enum FlowAudioRouteRecoveryPolicy {
public static func shouldRebuild(
reasonRaw: UInt,
formatIsStable: Bool,
engineIsRunning: Bool
) -> Bool {
switch AVAudioSession.RouteChangeReason(rawValue: reasonRaw) {
case .newDeviceAvailable, .oldDeviceUnavailable:
return true
case .categoryChange, .routeConfigurationChange:
return !formatIsStable || !engineIsRunning
default:
return false
}
}
}
public struct FlowAudioSessionSnapshot: Sendable, Equatable {
public let sampleRate: Double
public let inputChannels: Int
public let inputPortType: String
public let inputPortUID: String
}
public final class FlowAudioEngineHandle: @unchecked Sendable {
public let engine: AVAudioEngine
public init(_ engine: AVAudioEngine) {
self.engine = engine
}
}
public final class FlowAudioSessionCoordinator: @unchecked Sendable {
private struct CaptureActivation: Sendable {
let snapshot: FlowAudioSessionSnapshot
let preferredInputUID: String?
}
public enum Mode: Sendable, Equatable {
case inactive
case playback
case capture
}
public static let shared = FlowAudioSessionCoordinator()
private let session = AVAudioSession.sharedInstance()
private let queue = DispatchQueue(
label: "com.osgkeyboard.flow.audio-session",
qos: .userInitiated
)
private var mode: Mode = .inactive
private var active = false
private init() {}
public func activateCapture() async throws -> FlowAudioSessionSnapshot {
let activation: CaptureActivation = try await perform {
if self.mode != .capture {
try self.session.setCategory(
.playAndRecord,
mode: .measurement,
options: [.defaultToSpeaker, .allowBluetoothHFP, .mixWithOthers]
)
}
if !self.active {
try self.session.setActive(true, options: .notifyOthersOnDeactivation)
self.active = true
}
// When an HFP device is already connected, iOS can initially
// report the built-in mic and switch to HFP after recording has
// begun. Select the route before building AVAudioEngine so short
// utterances do not land entirely inside that negotiation window.
let hfp = self.session.availableInputs?.first(where: {
$0.portType == .bluetoothHFP
})
if self.session.currentRoute.inputs.first?.portType != .bluetoothHFP,
let hfp {
do {
try self.session.setPreferredInput(hfp)
} catch {
OSGDiag.log(
"preferred HFP input failed: \(error.localizedDescription)",
category: "flow"
)
}
}
self.mode = .capture
return CaptureActivation(
snapshot: self.makeSnapshot(),
preferredInputUID: hfp?.uid
)
}
return await waitForStableRoute(
initial: activation.snapshot,
preferredInputUID: activation.preferredInputUID
)
}
public func activatePlayback() async -> Bool {
do {
_ = try await perform {
// Once capture has established a stable playAndRecord route,
// keep that active session between utterances. The engine and
// tap are stopped separately, so the microphone is released,
// while avoiding a category flip that eventually yields !pri
// after repeated PiP/capture cycles.
if self.active, self.mode == .capture {
return self.makeSnapshot()
}
if self.mode != .playback {
try self.session.setCategory(
.playback,
mode: .moviePlayback,
options: [.mixWithOthers]
)
}
if !self.active {
try self.session.setActive(true)
self.active = true
}
self.mode = .playback
return self.makeSnapshot()
}
return true
} catch {
OSGDiag.log(
"PiP audio session failed: \(error.localizedDescription)",
category: "flow"
)
return false
}
}
public func deactivate() {
queue.async { [self] in
guard active else { return }
do {
try session.setActive(false, options: .notifyOthersOnDeactivation)
active = false
mode = .inactive
} catch {
OSGDiag.log(
"audio session deactivate failed: \(error.localizedDescription)",
category: "flow"
)
}
}
}
public func startEngine(_ handle: FlowAudioEngineHandle) async throws {
try await perform {
handle.engine.prepare()
try handle.engine.start()
return true
}
}
public func stopEngine(_ handle: FlowAudioEngineHandle) async {
_ = try? await perform {
if handle.engine.isRunning {
handle.engine.stop()
}
handle.engine.reset()
return true
}
}
public func enqueueStopEngine(_ handle: FlowAudioEngineHandle) {
queue.async {
if handle.engine.isRunning {
handle.engine.stop()
}
handle.engine.reset()
}
}
private func waitForStableRoute(
initial: FlowAudioSessionSnapshot,
preferredInputUID: String?,
timeout: TimeInterval = 1.5
) async -> FlowAudioSessionSnapshot {
let deadline = Date().addingTimeInterval(timeout)
var previous = initial
var stableReadCount = 0
while Date() < deadline {
try? await Task.sleep(nanoseconds: 50_000_000)
let current = await snapshot()
let preferredRouteReached = preferredInputUID == nil
|| current.inputPortUID == preferredInputUID
if current == previous, current.sampleRate > 0,
current.inputChannels > 0, preferredRouteReached {
stableReadCount += 1
} else {
stableReadCount = 0
}
if stableReadCount >= 4 {
return current
}
previous = current
}
return previous
}
private func snapshot() async -> FlowAudioSessionSnapshot {
(try? await perform { self.makeSnapshot() })
?? FlowAudioSessionSnapshot(
sampleRate: 0,
inputChannels: 0,
inputPortType: "none",
inputPortUID: ""
)
}
private func makeSnapshot() -> FlowAudioSessionSnapshot {
let input = session.currentRoute.inputs.first
return FlowAudioSessionSnapshot(
sampleRate: session.sampleRate,
inputChannels: session.inputNumberOfChannels,
inputPortType: input?.portType.rawValue ?? "none",
inputPortUID: input?.uid ?? ""
)
}
private func perform<T: Sendable>(
_ operation: @escaping @Sendable () throws -> T
) async throws -> T {
try await withCheckedThrowingContinuation { continuation in
queue.async {
continuation.resume(with: Result { try operation() })
}
}
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,612 @@
// LiveDictationController.swift
// OSGKeyboard · Shared
//
// Unified on-device dictation session: mic capture + iOS 26 SpeechAnalyzer.
// Used by the keyboard preview sheet, host-app dictation handoff, and any
// other foreground surface that needs live ASR without duplicating pipeline code.
//
// STATUS (v0.1.2): Retained as a "preview / one-shot handoff" path.
// The *primary* voice-session path is `FlowSessionManager` +
// `FlowContinuousCapture` (TypeWhisper-style continuous capture shared
// between host app and keyboard extension). The keyboard extension
// consumes results through `FlowSessionBridge`.
//
// This class is still imported by:
// - `OSGKeyboard/Views/PreviewASRController.swift` (typealias)
// - `OSGKeyboard/Views/KeyboardPreviewSheet.swift` (in-app preview)
// - `OSGKeyboard/Views/KeyboardPreviewSheet.swift` (host-app ASR preview)
// - `OSGKeyboardTests/PreviewASRControllerStateTests.swift`
//
// Do NOT remove without updating those call sites. The earlier
// `OSGKeyboardExt/Services/AudioCaptureService.swift` *was* a true
// dead duplicate and has been deleted (see AUDIT_APPSTORE.md P0-3).
// Owns its own AVAudioEngine + AVAudioSession, downsamples to 16 kHz
// mono Float32 on the audio thread (same as `AudioCaptureService`), and
// feeds `AudioBufferSnapshot` to the shared `ASRService` (the same
// pipeline the real keyboard extension
// uses, so the preview exercises the *real* iOS speech APIs, not a
// stub). Without this the in-app preview was a hardcoded transcript
// and "did you actually call SFSpeechRecognizer?" was a fair review
// note.
//
// Why not reuse `AudioCaptureService` from the extension? It lives in
// `OSGKeyboardExt`, an `app-extension` target the main app can't
// import its symbols. We could move it to `OSGKeyboardShared`, but
// `AVAudioSession` lifecycle differs enough between a keyboard
// extension (no background, no recording entitlement surprise) and a
// foreground app that a copy here is the lesser evil.
import Foundation
import AVFoundation
import Speech
import os
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
private enum LiveCaptureGatePhase: Equatable {
case idle
case recording
case draining
}
/// Thread-safe relay so the AVAudioEngine tap can yield snapshots without
/// hopping through `@MainActor` (which adds latency and can reorder frames).
private final class CaptureStreamRelay: @unchecked Sendable {
private let lock = OSAllocatedUnfairLock()
private var continuation: AsyncStream<AudioBufferSnapshot>.Continuation?
func bind(_ continuation: AsyncStream<AudioBufferSnapshot>.Continuation) {
lock.withLock { self.continuation = continuation }
}
func yield(_ snapshot: AudioBufferSnapshot) {
_ = lock.withLock { continuation?.yield(snapshot) }
}
func finish() {
lock.withLock {
continuation?.finish()
continuation = nil
}
}
}
@MainActor
public final class LiveDictationController: ObservableObject {
public enum Phase: Equatable {
case idle
case requestingPermission
case recording
case processing
case denied(String)
case error(String)
}
@Published public private(set) var phase: Phase = .idle
/// Normalized 0...1 RMS for the disc level meter. Polled from the
/// audio tap via `Task { @MainActor in ... }` the tap itself
/// runs on a real-time audio thread, so we never touch published
/// state from there.
@Published public private(set) var level: Double = 0
@Published public private(set) var currentPartial: String = ""
@Published public private(set) var errorMessage: String?
/// Set when a `.final` ASR event lands. The owning sheet observes
/// this and appends the text to its textbox, then clears it so the
/// next recording starts from zero.
@Published public var lastFinal: String = ""
private let asr: ASRService
private var audioEngine = AVAudioEngine()
/// `internal` (not `private`) so the regression test in
/// `OSGKeyboardTests/PreviewASRControllerStateTests.swift` can
/// install a known consumer task and assert `stop()` doesn't
/// cancel it. The class is `@MainActor`-isolated, so the
/// natural Swift 6 isolation rules still prevent production
/// code outside the class from racing on it.
public var asrTask: Task<Void, Never>?
private let streamRelay = CaptureStreamRelay()
private var chunkedPipeline: ChunkedUtterancePipeline?
private let captureGate = OSAllocatedUnfairLock(initialState: LiveCaptureGatePhase.idle)
private let drainTracker = FlowCaptureDrainTracker()
private var audioConverter: AVAudioConverter?
private var targetFormat: AVAudioFormat?
private var hwFormat: AVAudioFormat?
private var didConfigureAudioSession = false
private var didInstallTap = false
public init(asr: ASRService? = nil) {
self.asr = asr ?? ASRServiceFactory.make(store: AppGroupStore())
}
/// Start dictation using a persisted settings locale id (`auto`, `zh-Hans`, ).
public func start(localeId: String) async {
await start(locale: SpeechLocaleResolver.resolve(localeId))
}
public func start(locale: Locale) async {
// Re-entry guard: ignore taps that arrive while we're already
// running. (The sheet's `cyclePhase` is also guarded, but
// async race windows are easier to lock down here.)
switch phase {
case .recording, .requestingPermission, .processing:
return
default:
break
}
// Cancel any leftover consumer task from a previous recording.
// Normally `stop()` lets the task run to completion (so it can
// see the `.final` and transition out of `.processing`), but if
// the user smashed the disc twice stop, then immediately
// start the previous task might still be draining. Cancel it
// here so we don't have two consumer tasks fighting over the
// same `events` stream.
asrTask?.cancel()
asrTask = nil
if let pipeline = chunkedPipeline {
Task { await pipeline.cancel() }
}
chunkedPipeline = nil
teardownCapturePipeline()
phase = .requestingPermission
currentPartial = ""
lastFinal = ""
errorMessage = nil
level = 0
// 1. Microphone permission. The helper is `nonisolated` so the
// (iOS < 17) callback closure does not inherit `@MainActor`
// `AVAudioSession.requestRecordPermission` delivers on a TCC
// reply queue, and a `@MainActor`-inferred closure body there
// hits `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift`.
let micGranted = await Self.requestMicrophonePermission()
guard micGranted else {
phase = .denied(NSLocalizedString("keyboard.denied.mic", comment: ""))
return
}
// 2. Speech recognition permission. Same reasoning as above:
// the callback fires on TCC's reply queue, NOT the main queue.
let speechGranted = await Self.requestSpeechRecognitionPermission()
guard speechGranted else {
phase = .denied(NSLocalizedString("keyboard.denied.speech", comment: ""))
return
}
// 3. Audio session only configure once per process.
//
// Category is `.record` (not `.playAndRecord`) because the
// preview never plays back audio it just records from the
// mic and hands the buffers to `SpeechAnalyzer`. On the
// iOS Simulator, `.playAndRecord` requires the
// `AURemoteIO` Audio Unit's *output* side to also be
// enabled, but the simulator's "speaker" reports a 0 Hz
// hardware format, so `AURemoteIO::enable` fails with
// `kAudioUnitErr_FormatNotSupported` (-10851) and any
// subsequent `installTap` traps with "Failed to create tap
// due to format mismatch". `.record` skips the output
// side entirely, so the simulator can record.
//
// The real keyboard extension (`OSGKeyboardExt`) keeps
// `.playAndRecord` because it runs on a real device where
// the output side has a real hardware format, and may want
// to play click sounds / haptic feedback. Only the preview
// needs the simulator-friendly category.
if !didConfigureAudioSession {
do {
let session = AVAudioSession.sharedInstance()
try session.setCategory(.record,
mode: .measurement,
options: [])
try session.setActive(true, options: .notifyOthersOnDeactivation)
didConfigureAudioSession = true
} catch {
debug("audio session failed: \(error.localizedDescription)")
phase = .error(String.localizedStringWithFormat(
NSLocalizedString("preview.error.audioSession", comment: ""),
error.localizedDescription
))
return
}
}
// The route may have changed while the preview was closed. A fresh
// engine created after session activation avoids a stale input node.
audioEngine = AVAudioEngine()
// 4. Spin up the engine + ASR.
phase = .recording
startEngineAndASR(locale: locale)
}
public func stop() {
// Don't `asrTask?.cancel()` here see the comment in
// `startEngineAndASR` for the full rationale. Short version:
// cancelling the consumer task at the same moment we close the
// audio stream also triggers the producer's
// `continuation.onTermination self?.cancel()` cascade, which
// marks the producer's outer task as cancelled and skips the
// `.final` event. The UI is then left in `.processing` forever
// because no one schedules the transition out. The consumer
// task naturally exits when `events` finishes, so the right
// thing is to let it run.
//
// If a previous `asrTask` is somehow still running (e.g. the
// user smashed the disc twice quickly), `start()` cancels it
// at the entry point as a safety net.
let partial = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
if !partial.isEmpty && lastFinal.isEmpty {
lastFinal = partial
currentPartial = ""
}
if phase == .recording {
phase = .processing
}
Task { @MainActor [weak self] in
await self?.drainTailAndTeardownCapture()
}
// Safety net: if the ASR pipeline never produces a `.final`
// (analyzer hang, system glitch, dropped continuation), force
// the UI back to idle after a short delay so the user isn't
// stuck. Normal recordings complete well under a second, so
// the 3-second budget is only hit on the unhappy path; if the
// pipeline finishes first and flips the phase to `.idle` (or
// `.error`), the check below no-ops.
Task { @MainActor [weak self] in
try? await Task.sleep(for: .seconds(3))
guard let self else { return }
if self.phase == .processing {
let stalePartial = self.currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
if !stalePartial.isEmpty, self.lastFinal.isEmpty {
self.debug("processing timeout, using partial")
self.lastFinal = stalePartial
self.currentPartial = ""
}
self.phase = .idle
}
}
}
public func reset() {
// Called by the sheet after appending `lastFinal` to the textbox,
// so the next recording can produce a fresh final without us
// double-appending.
lastFinal = ""
if phase == .processing {
phase = .idle
}
}
// MARK: - Engine + ASR
private func startEngineAndASR(locale: Locale) {
let inputNode = audioEngine.inputNode
let hwFormat = inputNode.inputFormat(forBus: 0)
// Pre-flight check: a placeholder / unconfigured input bus
// reports `sampleRate == 0` (or `channelCount == 0`).
// `installTap` on such a bus traps with "Failed to create
// tap due to format mismatch" (an NSException, not a Swift
// `Error`, so we can't `try`/`catch` it). The safest fix
// is to refuse the tap up front and surface a clear
// `.error` phase instead of crashing the app. We've seen
// this on the iOS Simulator when the host's microphone
// permission isn't granted to CoreSimulator, and on
// devices where the audio session is in an unexpected
// state from a previous foreground/background transition.
guard hwFormat.sampleRate > 0, hwFormat.channelCount > 0 else {
debug("invalid hardware format sr=\(hwFormat.sampleRate) ch=\(hwFormat.channelCount)")
phase = .error(
String.localizedStringWithFormat(
NSLocalizedString("preview.error.micUnavailable", comment: ""),
hwFormat.sampleRate,
Int(hwFormat.channelCount)
)
)
return
}
let targetSampleRate: Double = 16_000
guard let targetFormat = AVAudioFormat(
commonFormat: .pcmFormatFloat32,
sampleRate: targetSampleRate,
channels: 1,
interleaved: false
) else {
phase = .error(NSLocalizedString("preview.error.formatCreate", comment: ""))
return
}
guard let converter = AVAudioConverter(from: hwFormat, to: targetFormat) else {
debug("converter creation failed")
phase = .error(NSLocalizedString("preview.error.converterCreate", comment: ""))
return
}
audioConverter = converter
self.targetFormat = targetFormat
self.hwFormat = hwFormat
drainTracker.reset()
captureGate.withLock { $0 = .recording }
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
streamRelay.bind(continuation)
// Tap the hardware input. The closure passed to `installTap` runs
// on the AVAudioEngine real-time audio thread. In Swift 6 strict
// concurrency, a closure literal defined inside a `@MainActor`
// method inherits `@MainActor` isolation, which would trip
// `dispatch_assert_queue_fail` on first invocation from the
// audio thread. The fix is to build the actual tap body in a
// `nonisolated` helper (`makeAudioTapBlock`) and have the
// installTap closure be a single function reference function
// references never carry inferred isolation, so the dispatch
// runtime is happy and the body runs wherever AVAudioEngine
// wants it (the audio thread).
let onMeter: @Sendable (Double) -> Void = { [weak self] meter in
Task { @MainActor [weak self] in
guard let self else { return }
// Lightweight smoothing so the disc ring doesn't jitter.
self.level = self.level * 0.55 + meter * 0.45
}
}
let relay = streamRelay
let gate = captureGate
let tracker = drainTracker
let policy = FlowCaptureTailDrainPolicy.flowDefault
let onSnapshot: @Sendable (AudioBufferSnapshot) -> Void = { snapshot in
let phase = gate.withLock { $0 }
guard phase == .recording || phase == .draining else { return }
relay.yield(snapshot)
if phase == .draining {
tracker.noteAudio(samples: snapshot.samples, policy: policy)
}
}
let tap = Self.makeAudioTapBlock(
converter: converter,
targetFormat: targetFormat,
hwFormat: hwFormat,
onMeter: onMeter,
onSnapshot: onSnapshot
)
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
didInstallTap = true
audioEngine.prepare()
do {
try audioEngine.start()
} catch {
debug("audio engine start failed: \(error.localizedDescription)")
phase = .error(String.localizedStringWithFormat(
NSLocalizedString("preview.error.engineStart", comment: ""),
error.localizedDescription
))
return
}
// 5. Pipelined ASR (same chunk path as Flow host).
let pipeline = ChunkedUtterancePipeline(asr: asr, locale: locale)
chunkedPipeline = pipeline
asrTask = Task.detached(priority: .userInitiated) { [weak controller = self] in
let outcome = await pipeline.transcribe(stream: stream) { partial in
Task { @MainActor in
controller?.currentPartial = partial
}
}
// Re-bind `controller` inside the `@MainActor` block so the
// weak reference is captured under the right isolation. Swift
// 6 strict concurrency otherwise complains about a
// task-isolated reference escaping into a main-actor closure.
await MainActor.run { [weak controller] in
guard let controller else { return }
switch outcome {
case .success(let success):
let trimmed = success.text.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty {
controller.lastFinal = trimmed
controller.currentPartial = ""
}
if controller.phase == .processing || controller.phase == .recording {
controller.phase = .idle
}
case .failure(let message):
controller.debug("asr error: \(message)")
controller.teardownCapturePipeline()
controller.errorMessage = message
controller.phase = .error(message)
case .cancelled:
if controller.phase == .processing {
controller.phase = .idle
}
}
}
}
}
// MARK: - Permission helpers (nonisolated)
//
// `SFSpeechRecognizer.requestAuthorization` delivers its callback
// on a TCC reply queue, NOT the main queue. If we wrap that
// callback inline in `start(locale:)` which is `@MainActor`
// Swift 6 strict concurrency infers the closure body as
// `@MainActor`, and the runtime crashes on
// `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift` as
// soon as TCC calls us back.
//
// The first attempt (commit `e8a0310`) extracted the entire
// permission request into a `nonisolated static func` helper.
// That worked in isolation, but the Swift 6 optimizer
// inlined those helpers back into `start(locale:)`. After
// inlining, the `withCheckedContinuation` body and the
// `requestAuthorization` callback were re-typed in the
// `@MainActor` context of the caller, and the runtime
// assertion came right back same crash, different symbol:
// `closure #1 in closure #2 in PreviewASRController.start(locale:)`.
//
// The fix that survives inlining is the *function-reference*
// pattern, the same one used for `installTap` in
// `makeAudioTapBlock` below. The callback is built in a
// `nonisolated` static helper that takes a `CheckedContinuation`
// and returns the `(Status) -> Void` handler. The body of that
// helper has no enclosing actor, so the closure is created in
// nonisolated context. When TCC calls us back, the runtime
// sees a nonisolated closure on a non-main queue and is happy.
//
// `cont.resume(...)` is itself thread-safe on
// `CheckedContinuation`, so we don't need to hop back to the
// main actor before resuming.
private nonisolated static func requestMicrophonePermission() async -> Bool {
// iOS 17+ API; the iOS < 17 fallback (`AVAudioSession.recordPermission`
// + `requestRecordPermission` callback) is gone now that the
// deployment target is iOS 26.
switch AVAudioApplication.shared.recordPermission {
case .granted: return true
case .denied: return false
case .undetermined: return await AVAudioApplication.requestRecordPermission()
@unknown default: return false
}
}
private nonisolated static func requestSpeechRecognitionPermission() async -> Bool {
await withCheckedContinuation { (cont: CheckedContinuation<Bool, Never>) in
SFSpeechRecognizer.requestAuthorization(
Self.makeSpeechAuthHandler(continuation: cont)
)
}
}
private nonisolated static func makeSpeechAuthHandler(
continuation: CheckedContinuation<Bool, Never>
) -> @Sendable (SFSpeechRecognizerAuthorizationStatus) -> Void {
return { status in
continuation.resume(returning: status == .authorized)
}
}
// MARK: - Audio tap (nonisolated, runs on AVAudioEngine render thread)
//
// `AVAudioNode.installTap`'s callback fires on the audio engine's
// real-time render thread. In Swift 6 strict concurrency, a closure
// literal defined inside a `@MainActor` method inherits `@MainActor`
// isolation and `dispatch_assert_queue_fail` fires the moment
// the runtime tries to dispatch that closure on a non-main queue.
//
// The trick is to build the actual tap body in a `nonisolated`
// function and have the installTap closure be a *function reference*
// to that helper. Function references never carry inferred
// isolation, so the dispatch runtime is satisfied and the body
// runs wherever AVAudioEngine wants. State updates to
// `self.level` and the AsyncStream continuation hop back to the
// main actor via `Task { @MainActor in }`, which is itself
// safe to call from a non-isolated context.
private nonisolated static func makeAudioTapBlock(
converter: AVAudioConverter,
targetFormat: AVAudioFormat,
hwFormat: AVAudioFormat,
onMeter: @Sendable @escaping (Double) -> Void,
onSnapshot: @Sendable @escaping (AudioBufferSnapshot) -> Void
) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
// `@Sendable` on the returned closure makes the Sendable
// conformance explicit. `AVAudioNodeTapBlock` is declared as
// a plain escaping closure in the SDK; we cast at the call
// site via `as @Sendable`.
return { buffer, _ in
// 1) Level meter from raw hardware buffer.
let n = Int(buffer.frameLength)
var sumSquares: Float = 0
if let channelData = buffer.floatChannelData?[0], n > 0 {
for i in 0..<n {
let v = channelData[i]
sumSquares += v * v
}
}
let rms = n > 0 ? sqrtf(sumSquares / Float(n)) : 0
let meter = min(Double(rms) * 4.0, 1.0)
onMeter(meter)
// 2) Downsample to 16 kHz mono Float32 for ASR (matches
// `AudioCaptureService` and Apple's `considering:` hint).
let outFrames = AVAudioFrameCount(
Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate
)
guard outFrames > 0,
let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outFrames)
else { return }
var error: NSError?
let status = converter.convert(to: outBuffer, error: &error) { _, outStatus in
outStatus.pointee = .haveData
return buffer
}
guard status == .haveData, error == nil, outBuffer.frameLength > 0 else { return }
let snapshot = AudioBufferSnapshot(buffer: outBuffer)
guard !snapshot.samples.isEmpty else { return }
onSnapshot(snapshot)
}
}
private func drainTailAndTeardownCapture() async {
let beganDrain = captureGate.withLock { phase -> Bool in
switch phase {
case .recording:
phase = .draining
return true
case .draining, .idle:
return false
}
}
guard beganDrain else { return }
drainTracker.beginDrain()
let policy = FlowCaptureTailDrainPolicy.flowDefault
_ = await FlowUtteranceEndCoordinator.awaitTailCapture(
tracker: drainTracker,
policy: policy
)
// Trailing speech is preserved by the live `.draining` forwarding
// loop above. We deliberately do NOT signal `.endOfStream` to the
// converter to squeeze its internal filter tail: that both races the
// still-running audio-thread tap on the same non-thread-safe converter
// and (in reused-converter paths) permanently locks it. The dropped
// tail is sub-millisecond and inaudible.
streamRelay.finish()
teardownCaptureEngine()
captureGate.withLock { $0 = .idle }
drainTracker.reset()
audioConverter = nil
targetFormat = nil
hwFormat = nil
try? AVAudioSession.sharedInstance().setActive(
false,
options: .notifyOthersOnDeactivation
)
}
private func teardownCaptureEngine() {
if didInstallTap {
audioEngine.inputNode.removeTap(onBus: 0)
didInstallTap = false
}
if audioEngine.isRunning {
audioEngine.stop()
}
}
private func teardownCapturePipeline() {
teardownCaptureEngine()
streamRelay.finish()
}
private func debug(_ message: String) {
#if DEBUG
print("🎙️[LiveDictationController] \(message)")
#endif
}
}
@@ -0,0 +1,126 @@
// TipPurchaseManager.swift
// OSGKeyboard · Shared
//
// Optional ¥30 consumable tip via StoreKit 2. Voluntary support only
// no feature gates, no App Group sync, no restore (Apple consumable rules).
import Foundation
import Combine
import StoreKit
#if canImport(OSGKeyboardShared)
import OSGKeyboardShared
#endif
public enum TipPurchaseState: Equatable {
case idle
case loading
case purchasing
case succeeded
case failed(String)
}
@MainActor
public final class TipPurchaseManager: ObservableObject {
public static let shared = TipPurchaseManager()
@Published public private(set) var product: Product?
@Published public private(set) var purchaseState: TipPurchaseState = .idle
@Published public private(set) var supportCount: Int
private var transactionUpdatesTask: Task<Void, Never>?
public init(defaults: UserDefaults = .standard) {
supportCount = defaults.integer(forKey: TipProduct.supportCountDefaultsKey)
transactionUpdatesTask = Task { [weak self] in
await self?.listenForTransactionUpdates()
}
Task { await loadProducts() }
}
deinit {
transactionUpdatesTask?.cancel()
}
/// Loads the tip product from the App Store / StoreKit Test configuration.
public func loadProducts() async {
guard purchaseState != .purchasing else { return }
purchaseState = .loading
do {
let products = try await Product.products(for: TipProduct.allProductIDs)
product = products.first(where: { $0.id == TipProduct.supportID })
purchaseState = product == nil
? .failed(SharedL10n.string("tip.error.productUnavailable"))
: .idle
} catch {
product = nil
purchaseState = .failed(error.localizedDescription)
}
}
/// Starts the StoreKit purchase sheet for the support tip.
public func purchase() async {
guard let product else {
purchaseState = .failed(SharedL10n.string("tip.error.productUnavailable"))
return
}
purchaseState = .purchasing
do {
let result = try await product.purchase()
switch result {
case .success(let verification):
let transaction = try Self.verified(verification)
await handleCompletedTip(transaction)
purchaseState = .succeeded
case .userCancelled:
purchaseState = .idle
case .pending:
purchaseState = .failed(SharedL10n.string("tip.error.pending"))
@unknown default:
purchaseState = .failed(SharedL10n.string("tip.error.unknown"))
}
} catch {
purchaseState = .failed(error.localizedDescription)
}
}
/// Clears transient success / failure state after the UI acknowledges it.
public func acknowledgePurchaseState() {
switch purchaseState {
case .succeeded, .failed:
purchaseState = .idle
default:
break
}
}
private func listenForTransactionUpdates() async {
for await update in Transaction.updates {
guard let transaction = try? Self.verified(update),
transaction.productID == TipProduct.supportID else { continue }
await handleCompletedTip(transaction)
}
}
private func handleCompletedTip(_ transaction: Transaction) async {
await transaction.finish()
recordSupportPurchase()
}
private func recordSupportPurchase() {
supportCount += 1
UserDefaults.standard.set(supportCount, forKey: TipProduct.supportCountDefaultsKey)
}
private static func verified<T>(_ result: VerificationResult<T>) throws -> T {
switch result {
case .verified(let value):
return value
case .unverified:
throw TipPurchaseError.failedVerification
}
}
}
private enum TipPurchaseError: Error {
case failedVerification
}