feat(keyboard): improve typing, voice flow, and polish reliability
Reduce extension memory pressure and delivery races while adding richer candidates, tactile feedback, and safer two-level creative polishing.
This commit is contained in:
@@ -0,0 +1,161 @@
|
||||
// SevenDayUsageChart.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// 7-day dictation bar chart. Platform shells wrap this in their own page
|
||||
// layout; the chart itself only needs points + UI language.
|
||||
|
||||
import Charts
|
||||
import SwiftUI
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public struct SevenDayUsageChart: View {
|
||||
@Environment(\.themePalette) private var palette
|
||||
|
||||
public let points: [UsageStatisticsStore.DailyUsagePoint]
|
||||
public let language: AppUILanguage
|
||||
/// Bar area height. Phone stacked layout uses a shorter value.
|
||||
public var chartMinHeight: CGFloat
|
||||
/// When true, wrap content in `UsageSurfaceCard` (default). Pass false if
|
||||
/// the caller already provides a surface.
|
||||
public var embedsInCard: Bool
|
||||
/// When true (iPad split), the chart fills all available height. When false
|
||||
/// (phone stacked home), the bar area is a fixed `chartMinHeight` so the card
|
||||
/// stays compact and does not steal space from surrounding content.
|
||||
public var expands: Bool
|
||||
|
||||
public init(
|
||||
points: [UsageStatisticsStore.DailyUsagePoint],
|
||||
language: AppUILanguage,
|
||||
chartMinHeight: CGFloat = 96,
|
||||
embedsInCard: Bool = true,
|
||||
expands: Bool = true
|
||||
) {
|
||||
self.points = points
|
||||
self.language = language
|
||||
self.chartMinHeight = chartMinHeight
|
||||
self.embedsInCard = embedsInCard
|
||||
self.expands = expands
|
||||
}
|
||||
|
||||
private var total: Int {
|
||||
points.reduce(0) { $0 + $1.value }
|
||||
}
|
||||
|
||||
private var maxValue: Int {
|
||||
max(points.map(\.value).max() ?? 0, 1)
|
||||
}
|
||||
|
||||
private var chartLocale: Locale {
|
||||
Locale(identifier: language.resolvedLanguageCode())
|
||||
}
|
||||
|
||||
/// The real day for each bar; used to decide which axis marks get a label.
|
||||
private var pointDates: Set<Date> {
|
||||
Set(points.map(\.date))
|
||||
}
|
||||
|
||||
/// Axis tick positions: the 7 real days plus one trailing boundary (last + 1
|
||||
/// day) so `centered: true` has a span to center the final day's label within.
|
||||
private var axisDates: [Date] {
|
||||
let dates = points.map(\.date)
|
||||
guard let last = dates.last,
|
||||
let boundary = Calendar.current.date(byAdding: .day, value: 1, to: last)
|
||||
else { return dates }
|
||||
return dates + [boundary]
|
||||
}
|
||||
|
||||
public var body: some View {
|
||||
Group {
|
||||
if embedsInCard {
|
||||
UsageSurfaceCard(padding: Spacing.md) {
|
||||
chartContent
|
||||
}
|
||||
} else {
|
||||
chartContent
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var chartContent: some View {
|
||||
VStack(alignment: .leading, spacing: Spacing.sm) {
|
||||
header
|
||||
chart
|
||||
.frame(maxWidth: .infinity, maxHeight: expands ? .infinity : nil)
|
||||
}
|
||||
.frame(
|
||||
maxWidth: .infinity,
|
||||
maxHeight: expands ? .infinity : nil,
|
||||
alignment: .topLeading
|
||||
)
|
||||
}
|
||||
|
||||
// MARK: - Header
|
||||
|
||||
private var header: some View {
|
||||
HStack(alignment: .firstTextBaseline) {
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
Text(SharedL10n.string("stat.weekChart.title", language: language).uppercased())
|
||||
.font(TypeStyle.caption2)
|
||||
.tracking(0.6)
|
||||
.foregroundStyle(palette.textTertiary)
|
||||
Text(SharedL10n.string("stat.weekChart.caption", language: language))
|
||||
.font(TypeStyle.caption)
|
||||
.foregroundStyle(palette.textSecondary)
|
||||
}
|
||||
Spacer(minLength: Spacing.sm)
|
||||
Text(UsageStatisticsStore.formatCount(total, language: language))
|
||||
.font(TypeStyle.title2)
|
||||
.foregroundStyle(palette.accent)
|
||||
.lineLimit(1)
|
||||
.minimumScaleFactor(0.7)
|
||||
.contentTransition(.numericText())
|
||||
.animation(Motion.soft, value: total)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Bars
|
||||
|
||||
@ViewBuilder
|
||||
private var chart: some View {
|
||||
if total == 0 {
|
||||
Text(SharedL10n.string("stat.weekChart.empty", language: language))
|
||||
.font(TypeStyle.caption)
|
||||
.foregroundStyle(palette.textTertiary)
|
||||
.frame(maxWidth: .infinity, maxHeight: expands ? .infinity : nil, alignment: .center)
|
||||
.multilineTextAlignment(.center)
|
||||
.frame(height: expands ? nil : chartMinHeight)
|
||||
.frame(minHeight: expands ? chartMinHeight : nil)
|
||||
} else {
|
||||
Chart(points) { point in
|
||||
BarMark(
|
||||
x: .value("day", point.date, unit: .day),
|
||||
y: .value("chars", point.value)
|
||||
)
|
||||
.cornerRadius(4)
|
||||
.foregroundStyle(palette.accent.gradient)
|
||||
}
|
||||
.chartYScale(domain: 0...max(1, Int(ceil(Double(maxValue) * 1.15))))
|
||||
.chartYAxis(.hidden)
|
||||
.chartXAxis {
|
||||
// Center each weekday label under its bar. Date `BarMark`s draw the
|
||||
// bar centered within its day band, but axis labels default to the
|
||||
// day's leading tick, so `centered: true` re-centers them on the bar.
|
||||
//
|
||||
// `centered` positions a label between its tick and the *next* tick,
|
||||
// so the final day (Saturday) would be dropped for lack of a trailing
|
||||
// tick. We append one boundary tick (last day + 1) to give it a span,
|
||||
// and only draw labels for the real 7 days.
|
||||
AxisMarks(values: axisDates) { value in
|
||||
if let date = value.as(Date.self), pointDates.contains(date) {
|
||||
AxisValueLabel(format: .dateTime.weekday(.narrow), centered: true)
|
||||
}
|
||||
}
|
||||
}
|
||||
.environment(\.locale, chartLocale)
|
||||
.frame(height: expands ? nil : chartMinHeight)
|
||||
.frame(minHeight: expands ? chartMinHeight : nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
// SupportDeveloperSection.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Optional voluntary tip block for Settings. Does not gate features.
|
||||
|
||||
import SwiftUI
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
/// iOS Settings card for the consumable support tip.
|
||||
public struct SupportDeveloperSection: View {
|
||||
@ObservedObject private var tipManager: TipPurchaseManager
|
||||
@Environment(\.themePalette) private var palette
|
||||
|
||||
private let language: AppUILanguage
|
||||
|
||||
@State private var showThankYouAlert = false
|
||||
@State private var showErrorAlert = false
|
||||
@State private var errorMessage = ""
|
||||
|
||||
public init(
|
||||
language: AppUILanguage,
|
||||
tipManager: TipPurchaseManager = .shared
|
||||
) {
|
||||
self.language = language
|
||||
self.tipManager = tipManager
|
||||
}
|
||||
|
||||
public var body: some View {
|
||||
CardSection(title: SharedL10n.string("tip.title", language: language)) {
|
||||
VStack(alignment: .leading, spacing: Spacing.sm) {
|
||||
SupportDeveloperTipBody(language: language)
|
||||
|
||||
if tipManager.supportCount > 0 {
|
||||
Text(
|
||||
SharedL10n.format(
|
||||
"tip.thankYou.past",
|
||||
language: language,
|
||||
tipManager.supportCount
|
||||
)
|
||||
)
|
||||
.font(TypeStyle.caption)
|
||||
.foregroundStyle(palette.accent)
|
||||
}
|
||||
|
||||
tipButton
|
||||
.disabled(isPurchaseInFlight)
|
||||
|
||||
Text(SharedL10n.string("tip.consumableNotice", language: language))
|
||||
.font(TypeStyle.caption2)
|
||||
.foregroundStyle(palette.textTertiary)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
.padding(Spacing.md)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.surfaceCard()
|
||||
}
|
||||
.onChange(of: tipManager.purchaseState) { _, newValue in
|
||||
switch newValue {
|
||||
case .succeeded:
|
||||
showThankYouAlert = true
|
||||
case .failed(let message):
|
||||
errorMessage = message
|
||||
showErrorAlert = true
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
.alert(
|
||||
SharedL10n.string("tip.thankYou.title", language: language),
|
||||
isPresented: $showThankYouAlert
|
||||
) {
|
||||
Button(SharedL10n.string("tip.alert.dismiss", language: language)) {
|
||||
tipManager.acknowledgePurchaseState()
|
||||
}
|
||||
} message: {
|
||||
Text(SharedL10n.string("tip.thankYou.message", language: language))
|
||||
}
|
||||
.alert(
|
||||
SharedL10n.string("tip.error.title", language: language),
|
||||
isPresented: $showErrorAlert
|
||||
) {
|
||||
Button(SharedL10n.string("tip.alert.dismiss", language: language)) {
|
||||
tipManager.acknowledgePurchaseState()
|
||||
}
|
||||
} message: {
|
||||
Text(errorMessage)
|
||||
}
|
||||
}
|
||||
|
||||
private var isPurchaseInFlight: Bool {
|
||||
switch tipManager.purchaseState {
|
||||
case .loading, .purchasing:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
private var tipButton: some View {
|
||||
Button {
|
||||
Task { await tipManager.purchase() }
|
||||
} label: {
|
||||
HStack(spacing: Spacing.sm) {
|
||||
if isPurchaseInFlight {
|
||||
ProgressView()
|
||||
.controlSize(.small)
|
||||
}
|
||||
Text(tipButtonTitle)
|
||||
.font(TypeStyle.body.weight(.semibold))
|
||||
}
|
||||
.frame(maxWidth: .infinity)
|
||||
.padding(.vertical, Spacing.sm)
|
||||
.foregroundStyle(.white)
|
||||
.background(palette.accent, in: RoundedRectangle(cornerRadius: Radius.large, style: .continuous))
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.accessibilityLabel(tipButtonTitle)
|
||||
}
|
||||
|
||||
private var tipButtonTitle: String {
|
||||
if let product = tipManager.product {
|
||||
return SharedL10n.format("tip.button", language: language, product.displayPrice)
|
||||
}
|
||||
return SharedL10n.string("tip.buttonFallback", language: language)
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared copy + tip button for macOS Settings (Mac chrome wraps this).
|
||||
public struct SupportDeveloperTipBody: View {
|
||||
@Environment(\.themePalette) private var palette
|
||||
|
||||
private let language: AppUILanguage
|
||||
|
||||
public init(language: AppUILanguage) {
|
||||
self.language = language
|
||||
}
|
||||
|
||||
public var body: some View {
|
||||
Text(SharedL10n.string("tip.body", language: language))
|
||||
.font(TypeStyle.body)
|
||||
.foregroundStyle(palette.textSecondary)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
// UsageStatsCluster.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Cross-platform home / dashboard stats: 7-day chart + cumulative metrics.
|
||||
// Callers observe their store and pass plain values — Shared stays unbound
|
||||
// from platform singletons.
|
||||
|
||||
import SwiftUI
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public struct UsageStatsCluster: View {
|
||||
@Environment(\.themePalette) private var palette
|
||||
|
||||
public enum Layout: Sendable, Equatable {
|
||||
/// Chart left, 2×2 `UsageStatCard` grid right (Mac / iPad).
|
||||
case split
|
||||
/// Chart above a compact single-card 2×2 grid (iPhone).
|
||||
case stacked
|
||||
}
|
||||
|
||||
/// 手机端 2×2 统计网格的紧凑固定高度(沿用旧版 HomeStatsCard 数值)。
|
||||
static let compactGridHeight: CGFloat = 166
|
||||
|
||||
public let layout: Layout
|
||||
public let language: AppUILanguage
|
||||
public let points: [UsageStatisticsStore.DailyUsagePoint]
|
||||
public let dictationCharacterCount: Int
|
||||
public let dictationDurationSeconds: TimeInterval
|
||||
public let translationCharacterCount: Int
|
||||
public let dictionaryTermCount: Int
|
||||
/// 小屏(如 iPhone SE)收紧 stacked 图表高度,把空间让给下方的输入框。
|
||||
public let compact: Bool
|
||||
|
||||
public init(
|
||||
layout: Layout,
|
||||
language: AppUILanguage,
|
||||
points: [UsageStatisticsStore.DailyUsagePoint],
|
||||
dictationCharacterCount: Int,
|
||||
dictationDurationSeconds: TimeInterval,
|
||||
translationCharacterCount: Int,
|
||||
dictionaryTermCount: Int,
|
||||
compact: Bool = false
|
||||
) {
|
||||
self.layout = layout
|
||||
self.language = language
|
||||
self.points = points
|
||||
self.dictationCharacterCount = dictationCharacterCount
|
||||
self.dictationDurationSeconds = dictationDurationSeconds
|
||||
self.translationCharacterCount = translationCharacterCount
|
||||
self.dictionaryTermCount = dictionaryTermCount
|
||||
self.compact = compact
|
||||
}
|
||||
|
||||
public var body: some View {
|
||||
switch layout {
|
||||
case .split:
|
||||
splitBody
|
||||
case .stacked:
|
||||
stackedBody
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Split (Mac / iPad)
|
||||
|
||||
private var splitBody: some View {
|
||||
HStack(alignment: .top, spacing: Spacing.md) {
|
||||
SevenDayUsageChart(points: points, language: language)
|
||||
.frame(maxWidth: .infinity, maxHeight: .infinity)
|
||||
splitStatGrid
|
||||
.frame(maxWidth: .infinity)
|
||||
}
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
|
||||
private var splitStatGrid: some View {
|
||||
VStack(spacing: Spacing.md) {
|
||||
HStack(spacing: Spacing.md) {
|
||||
UsageStatCard(
|
||||
title: SharedL10n.string("stat.words", language: language),
|
||||
value: UsageStatisticsStore.formatCount(dictationCharacterCount, language: language),
|
||||
caption: SharedL10n.string("stat.transcribed", language: language),
|
||||
systemImage: "text.alignleft",
|
||||
accent: true
|
||||
)
|
||||
UsageStatCard(
|
||||
title: SharedL10n.string("stat.dictationTime", language: language),
|
||||
value: UsageStatisticsStore.formatDuration(dictationDurationSeconds, language: language),
|
||||
caption: SharedL10n.string("stat.cumulativeDuration", language: language),
|
||||
systemImage: "waveform"
|
||||
)
|
||||
}
|
||||
HStack(spacing: Spacing.md) {
|
||||
UsageStatCard(
|
||||
title: SharedL10n.string("stat.translation", language: language),
|
||||
value: UsageStatisticsStore.formatCount(translationCharacterCount, language: language),
|
||||
caption: SharedL10n.string("stat.cumulativeTranslation", language: language),
|
||||
systemImage: "character.bubble"
|
||||
)
|
||||
UsageStatCard(
|
||||
title: SharedL10n.string("stat.dictionary", language: language),
|
||||
value: UsageStatisticsStore.formatCount(dictionaryTermCount, language: language),
|
||||
caption: SharedL10n.string("stat.customTerms", language: language),
|
||||
systemImage: "character.book.closed"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Stacked (iPhone)
|
||||
|
||||
private var stackedBody: some View {
|
||||
VStack(spacing: compact ? Spacing.sm : Spacing.md) {
|
||||
SevenDayUsageChart(
|
||||
points: points,
|
||||
language: language,
|
||||
chartMinHeight: compact ? 72 : 96,
|
||||
expands: false
|
||||
)
|
||||
compactStatGrid
|
||||
}
|
||||
}
|
||||
|
||||
/// Phone-friendly 2×2: value + label only, single card with hairline dividers.
|
||||
private var compactStatGrid: some View {
|
||||
VStack(spacing: 0) {
|
||||
HStack(spacing: 0) {
|
||||
compactCell(
|
||||
systemImage: "waveform",
|
||||
value: UsageStatisticsStore.formatDuration(dictationDurationSeconds, language: language),
|
||||
label: SharedL10n.string("stat.dictationTime", language: language)
|
||||
)
|
||||
compactDivider
|
||||
compactCell(
|
||||
systemImage: "text.alignleft",
|
||||
value: UsageStatisticsStore.formatCount(dictationCharacterCount, language: language),
|
||||
label: SharedL10n.string("stat.words", language: language)
|
||||
)
|
||||
}
|
||||
Rectangle()
|
||||
.fill(palette.divider)
|
||||
.frame(height: 0.5)
|
||||
HStack(spacing: 0) {
|
||||
compactCell(
|
||||
systemImage: "character.bubble",
|
||||
value: UsageStatisticsStore.formatCount(translationCharacterCount, language: language),
|
||||
label: SharedL10n.string("stat.translation", language: language)
|
||||
)
|
||||
compactDivider
|
||||
compactCell(
|
||||
systemImage: "character.book.closed",
|
||||
value: UsageStatisticsStore.formatCount(dictionaryTermCount, language: language),
|
||||
label: SharedL10n.string("stat.dictionary", language: language)
|
||||
)
|
||||
}
|
||||
}
|
||||
// 锁定紧凑固定高度(对齐旧版 HomeStatsCard 的 166pt),避免格子按内容撑高。
|
||||
.frame(height: UsageStatsCluster.compactGridHeight)
|
||||
.background(palette.surface)
|
||||
.clipShape(RoundedRectangle(cornerRadius: Radius.xl, style: .continuous))
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: Radius.xl, style: .continuous)
|
||||
.stroke(palette.divider, lineWidth: 0.5)
|
||||
)
|
||||
}
|
||||
|
||||
private var compactDivider: some View {
|
||||
Rectangle()
|
||||
.fill(palette.divider)
|
||||
.frame(width: 0.5)
|
||||
}
|
||||
|
||||
private func compactCell(systemImage: String, value: String, label: String) -> some View {
|
||||
HStack(alignment: .top, spacing: Spacing.xs) {
|
||||
VStack(alignment: .leading, spacing: Spacing.xs) {
|
||||
Text(value)
|
||||
.font(.system(size: 24, weight: .semibold, design: .rounded))
|
||||
.foregroundStyle(palette.textPrimary)
|
||||
.lineLimit(1)
|
||||
.minimumScaleFactor(0.75)
|
||||
.contentTransition(.numericText())
|
||||
.animation(Motion.soft, value: value)
|
||||
Text(label)
|
||||
.font(TypeStyle.caption2)
|
||||
.foregroundStyle(palette.textSecondary)
|
||||
.lineLimit(1)
|
||||
.minimumScaleFactor(0.85)
|
||||
}
|
||||
Spacer(minLength: Spacing.xs)
|
||||
Image(systemName: systemImage)
|
||||
.font(.system(size: 18, weight: .semibold))
|
||||
.foregroundStyle(palette.accent)
|
||||
.padding(.top, 2)
|
||||
}
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.padding(Spacing.md)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleDevelopmentRegion</key>
|
||||
<string>$(DEVELOPMENT_LANGUAGE)</string>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>$(EXECUTABLE_NAME)</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
|
||||
<key>CFBundleInfoDictionaryVersion</key>
|
||||
<string>6.0</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>$(PRODUCT_NAME)</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>FMWK</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>1.0</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>1</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,25 @@
|
||||
// AudioBufferSnapshot+AVFoundation.swift
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// AVAudioPCMBuffer → AudioBufferSnapshot copy helper. Kept out of Shared so
|
||||
// the keyboard extension does not link AVFoundation for this type alone.
|
||||
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
extension AudioBufferSnapshot {
|
||||
/// Construct from an `AVAudioPCMBuffer` by copying out the channel data.
|
||||
public init(buffer: AVAudioPCMBuffer) {
|
||||
guard let channelData = buffer.floatChannelData else {
|
||||
self.init(samples: [], sampleRate: buffer.format.sampleRate)
|
||||
return
|
||||
}
|
||||
let n = Int(buffer.frameLength)
|
||||
var copy = [Float](repeating: 0, count: n)
|
||||
memcpy(©, channelData[0], n * MemoryLayout<Float>.size)
|
||||
self.init(samples: copy, sampleRate: buffer.format.sampleRate)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// ASRChunkTranscribing.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Minimal ASR surface for pipelined utterance chunking. Keeps
|
||||
// `ChunkedUtterancePipeline` independent of iOS-only `SpeechAnalyzer`.
|
||||
|
||||
import Foundation
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public enum ASRChunkResult: Sendable, Equatable {
|
||||
case success(String)
|
||||
case failure(String)
|
||||
case cancelled
|
||||
}
|
||||
|
||||
/// One-shot chunk transcription used by `ChunkedUtterancePipeline`.
|
||||
public protocol ASRChunkTranscribing: Sendable {
|
||||
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult
|
||||
func cancel()
|
||||
func resetForNewUtterance()
|
||||
}
|
||||
|
||||
extension ASRChunkTranscribing {
|
||||
public func cancel() {}
|
||||
public func resetForNewUtterance() {}
|
||||
}
|
||||
@@ -0,0 +1,677 @@
|
||||
// ASRService.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Speech-to-text abstraction. As of iOS 26 being the minimum
|
||||
// deployment target, the only ASR backend is `SpeechAnalyzer` +
|
||||
// `DictationTranscriber` — always on-device, no cloud fallback, no
|
||||
// `requiresOnDevice` toggle. The previous legacy recognizer path is
|
||||
// gone; if a future platform ever needs it back,
|
||||
// reintroduce as a sibling class in `ASRServiceFactory.make()`.
|
||||
//
|
||||
// Lives in `OSGKeyboardShared` (not the keyboard extension target) so
|
||||
// that the host app's `KeyboardPreviewSheet` can run the same ASR
|
||||
// pipeline against real iOS audio — without it, the in-app preview
|
||||
// was a static mock that never actually called `SFSpeechRecognizer`,
|
||||
// and "did you actually wire up ASR?" was a fair review note.
|
||||
|
||||
import Foundation
|
||||
import AVFoundation
|
||||
import CoreMedia
|
||||
import Speech
|
||||
import os
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
// MARK: - Sendable conformance
|
||||
|
||||
// `AVAudioPCMBuffer` and `SpeechAnalyzer` are not Sendable. We only
|
||||
// ever access them serially — the PCM buffer is built and consumed
|
||||
// inside a single Task, and the analyzer is cancelled but never
|
||||
// shared concurrently — so an unchecked conformance is sound here.
|
||||
extension AVAudioPCMBuffer: @unchecked @retroactive Sendable {}
|
||||
|
||||
// MARK: - Protocol
|
||||
|
||||
public protocol ASRService: ASRChunkTranscribing, Sendable {
|
||||
/// Start a transcription session. The returned stream emits `.partial`
|
||||
/// updates and exactly one `.final` (or `.error`) before finishing.
|
||||
/// `SpeechAnalyzer` is always fully on-device, so there is no
|
||||
/// `requiresOnDevice` flag — that legacy cloud-fallback control
|
||||
/// doesn't apply to the iOS 26 `SpeechAnalyzer` path.
|
||||
func transcribe(
|
||||
stream: AsyncStream<AudioBufferSnapshot>,
|
||||
locale: Locale
|
||||
) -> AsyncStream<ASREvent>
|
||||
|
||||
/// Cancel any in-flight recognition and tear down its tasks.
|
||||
func cancel()
|
||||
|
||||
/// Clears cancellation / cached session state before a new utterance.
|
||||
func resetForNewUtterance()
|
||||
|
||||
/// Pre-load locale assets and analyzer format for lower first-chunk latency.
|
||||
func warmup(locale: Locale) async
|
||||
|
||||
/// Transcribe one PCM chunk (Flow pipelined path). Default wraps `transcribe(stream:)`.
|
||||
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult
|
||||
}
|
||||
|
||||
extension ASRService {
|
||||
public func resetForNewUtterance() {}
|
||||
|
||||
public func warmup(locale: Locale) async {}
|
||||
|
||||
public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
|
||||
guard !samples.isEmpty else { return .success("") }
|
||||
if Task.isCancelled { return .cancelled }
|
||||
|
||||
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
|
||||
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
|
||||
continuation.yield(snapshot)
|
||||
continuation.finish()
|
||||
|
||||
var lastPartial = ""
|
||||
var finalText = ""
|
||||
var failure: String?
|
||||
|
||||
for await event in transcribe(stream: stream, locale: locale) {
|
||||
if Task.isCancelled { return .cancelled }
|
||||
switch event {
|
||||
case .capability:
|
||||
break
|
||||
case .partial(let text):
|
||||
lastPartial = text
|
||||
case .final(let text):
|
||||
finalText = text
|
||||
case .error(let message):
|
||||
failure = message
|
||||
}
|
||||
}
|
||||
|
||||
if let failure {
|
||||
return .failure(failure)
|
||||
}
|
||||
let trimmed = finalText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !trimmed.isEmpty {
|
||||
return .success(trimmed)
|
||||
}
|
||||
let partial = lastPartial.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !partial.isEmpty {
|
||||
return .success(partial)
|
||||
}
|
||||
return .success("")
|
||||
}
|
||||
}
|
||||
|
||||
public enum ASREvent: Sendable, Equatable {
|
||||
/// Emitted exactly once at the start of every `transcribe` call, so
|
||||
/// the UI can flag non-on-device locales (e.g. ja-JP on devices that
|
||||
/// only ship on-device ASR for en/zh). The ASR session continues
|
||||
/// either way — we fall back to cloud automatically.
|
||||
case capability(onDeviceSupported: Bool)
|
||||
case partial(String)
|
||||
case final(String)
|
||||
case error(String)
|
||||
}
|
||||
|
||||
// MARK: - Factory
|
||||
|
||||
public enum ASRServiceFactory {
|
||||
/// Returns on-device SpeechAnalyzer for `local`, or the user's cloud
|
||||
/// ASR provider when `engineMode == "cloud"`.
|
||||
public static func make(store: any ConfigurationStore = AppGroupStore()) -> ASRService {
|
||||
if store.engineMode == "cloud" {
|
||||
return CloudASRService(store: store)
|
||||
}
|
||||
return SpeechAnalyzerASR()
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - PCM format conversion (testable helpers)
|
||||
//
|
||||
// Extracted from the audio-thread hot path so the scaling + clipping
|
||||
// math can be exercised in unit tests without instantiating the
|
||||
// full ASR pipeline. See `OSGKeyboardTests/ASRConversionTests.swift`.
|
||||
extension ASRServiceFactory {
|
||||
|
||||
/// Convert a Float32 PCM buffer (`-1.0...1.0`) to an Int16 PCM
|
||||
/// buffer (`-32768...32767`).
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - source: Pointer to `sourceCount` `Float` samples. May be
|
||||
/// `nil` when `sourceCount == 0`.
|
||||
/// - sourceCount: Number of samples to convert. A `0` count
|
||||
/// turns the call into a no-op regardless of the pointers.
|
||||
/// - destination: Pointer to at least `sourceCount` slots of
|
||||
/// `Int16`. May be `nil` when `sourceCount == 0`.
|
||||
///
|
||||
/// Per-sample: `Int16(round(clamp(s * 32767, -32768, 32767)))`.
|
||||
/// The explicit clip matters: without it, `s == 1.0` would map
|
||||
/// to `+32767` (fine) but `s == 1.5` (which can show up at the
|
||||
/// audio engine boundary under gain) would wrap to a negative
|
||||
/// value after the implicit Float→Int16 conversion. The
|
||||
/// `round()` (rather than truncate) preserves DC balance — `0.5`
|
||||
/// quantises to `+16384`, not `+16383`, matching what most audio
|
||||
/// DAW round-trips expect.
|
||||
static func convertFloat32ToInt16(
|
||||
source: UnsafePointer<Float>?,
|
||||
sourceCount: Int,
|
||||
destination: UnsafeMutablePointer<Int16>?
|
||||
) {
|
||||
guard sourceCount > 0, let source, let destination else { return }
|
||||
for i in 0..<sourceCount {
|
||||
let scaled = source[i] * 32767.0
|
||||
let clipped = Swift.max(-32768.0, Swift.min(32767.0, scaled))
|
||||
destination[i] = Int16(clipped.rounded())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - SpeechAnalyzer implementation (iOS 26+)
|
||||
|
||||
/// ASR backend that uses the iOS 26 `SpeechAnalyzer` + `DictationTranscriber`
|
||||
/// APIs. This engine is always fully on-device.
|
||||
final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
private var analyzer: SpeechAnalyzer?
|
||||
private var analyzerTask: Task<Void, Never>?
|
||||
private var analyzerFinished = false
|
||||
/// Reused across pipelined chunks within one utterance (assets + format).
|
||||
private var chunkPreparedLocaleID: String?
|
||||
private var chunkAnalyzerFormat: AVAudioFormat?
|
||||
|
||||
func resetForNewUtterance() {
|
||||
// Keep chunk format / asset cache warm across utterances in one Flow session.
|
||||
}
|
||||
|
||||
func invalidateChunkPreparationCache() {
|
||||
lock.withLock {
|
||||
chunkPreparedLocaleID = nil
|
||||
chunkAnalyzerFormat = nil
|
||||
}
|
||||
}
|
||||
|
||||
func warmup(locale: Locale) async {
|
||||
let warmupStartedAt = Date()
|
||||
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
|
||||
Self.debug("warmup locale unsupported requested=\(locale.identifier(.bcp47))")
|
||||
FlowTrace.warn(
|
||||
"asr.local.warmup.localeUnsupported",
|
||||
"requested=\(locale.identifier(.bcp47))"
|
||||
)
|
||||
return
|
||||
}
|
||||
let localeID = resolvedLocale.identifier(.bcp47)
|
||||
let cachedLocaleID = lock.withLock { chunkPreparedLocaleID }
|
||||
if cachedLocaleID == localeID, lock.withLock({ chunkAnalyzerFormat != nil }) {
|
||||
Self.debug("warmup cache hit locale=\(localeID)")
|
||||
FlowTrace.asr("local.warmup.cacheHit", "locale=\(localeID)")
|
||||
return
|
||||
}
|
||||
|
||||
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
|
||||
Self.debug(
|
||||
"warmup start locale=\(localeID) customLMEnabled=\(setup.customLanguageModelEnabled) " +
|
||||
"customLMAttached=\(setup.usesCustomLanguageModel) " +
|
||||
"clmState=\(Self.describeCLMState(setup.clmState))"
|
||||
)
|
||||
|
||||
FlowTrace.asr(
|
||||
"local.warmup.begin",
|
||||
"locale=\(localeID) customLM=\(setup.usesCustomLanguageModel ? 1 : 0) "
|
||||
+ "clmState=\(Self.describeCLMState(setup.clmState))"
|
||||
)
|
||||
do {
|
||||
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
|
||||
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
|
||||
compatibleWith: [setup.transcriber],
|
||||
considering: Self.captureFormat
|
||||
) else {
|
||||
Self.debug("warmup format unsupported locale=\(localeID)")
|
||||
FlowTrace.warn("asr.local.warmup.formatUnsupported", "locale=\(localeID)")
|
||||
return
|
||||
}
|
||||
lock.withLock {
|
||||
chunkPreparedLocaleID = localeID
|
||||
chunkAnalyzerFormat = format
|
||||
}
|
||||
Self.debug("warmup ready locale=\(localeID)")
|
||||
FlowTrace.asr(
|
||||
"local.warmup.ready",
|
||||
"locale=\(localeID) analyzerRate=\(Int(format.sampleRate)) "
|
||||
+ "elapsed=\(FlowTrace.seconds(since: warmupStartedAt))s"
|
||||
)
|
||||
} catch {
|
||||
Self.debug("warmup failed: \(error.localizedDescription)")
|
||||
FlowTrace.warn(
|
||||
"asr.local.warmup.failed",
|
||||
"locale=\(localeID) elapsed=\(FlowTrace.seconds(since: warmupStartedAt))s "
|
||||
+ "error=\(error.localizedDescription)"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
|
||||
guard !samples.isEmpty else { return .success("") }
|
||||
if Task.isCancelled { return .cancelled }
|
||||
|
||||
let startedAt = Date()
|
||||
let rms = Self.rms(of: samples)
|
||||
Self.debug(
|
||||
"chunk start samples=\(samples.count) rms=\(String(format: "%.4f", rms)) " +
|
||||
"locale=\(locale.identifier(.bcp47))"
|
||||
)
|
||||
do {
|
||||
let text = try await transcribeSamples(samples, locale: locale, reuseChunkPrep: true)
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
Self.debug(
|
||||
"chunk success textLen=\(trimmed.count) elapsed=\(Self.elapsed(startedAt))s " +
|
||||
"empty=\(trimmed.isEmpty)"
|
||||
)
|
||||
FlowTrace.transcript(
|
||||
"asr.local.chunk",
|
||||
trimmed,
|
||||
"engine=local samples=\(samples.count) rms=\(String(format: "%.4f", rms)) "
|
||||
+ "elapsed=\(Self.elapsed(startedAt))s locale=\(locale.identifier(.bcp47))"
|
||||
)
|
||||
return trimmed.isEmpty ? .success("") : .success(trimmed)
|
||||
} catch is CancellationError {
|
||||
Self.debug("chunk cancelled elapsed=\(Self.elapsed(startedAt))s")
|
||||
FlowTrace.asr("local.chunk.cancelled", "samples=\(samples.count)")
|
||||
return .cancelled
|
||||
} catch {
|
||||
Self.debug("chunk failed elapsed=\(Self.elapsed(startedAt))s error=\(error.localizedDescription)")
|
||||
FlowTrace.warn(
|
||||
"asr.local.chunk.failed",
|
||||
"samples=\(samples.count) rms=\(String(format: "%.4f", rms)) "
|
||||
+ "error=\(error.localizedDescription)"
|
||||
)
|
||||
return .failure(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
/// Analyze a single PCM buffer without the streaming `transcribe` wrapper.
|
||||
private func transcribeSamples(
|
||||
_ samples: [Float],
|
||||
locale: Locale,
|
||||
reuseChunkPrep: Bool
|
||||
) async throws -> String {
|
||||
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
|
||||
throw ASRChunkError.localeUnsupported
|
||||
}
|
||||
let localeID = resolvedLocale.identifier(.bcp47)
|
||||
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
|
||||
Self.debug(
|
||||
"chunk setup locale=\(localeID) customLMEnabled=\(setup.customLanguageModelEnabled) " +
|
||||
"customLMAttached=\(setup.usesCustomLanguageModel) " +
|
||||
"clmState=\(Self.describeCLMState(setup.clmState))"
|
||||
)
|
||||
|
||||
let analyzerFormat: AVAudioFormat
|
||||
let cachedPrep = lock.withLock { (chunkPreparedLocaleID, chunkAnalyzerFormat) }
|
||||
if reuseChunkPrep,
|
||||
cachedPrep.0 == localeID,
|
||||
let cached = cachedPrep.1 {
|
||||
analyzerFormat = cached
|
||||
Self.debug(
|
||||
"chunk using cached analyzer format sr=\(Int(cached.sampleRate)) " +
|
||||
"channels=\(cached.channelCount) common=\(cached.commonFormat.rawValue)"
|
||||
)
|
||||
} else {
|
||||
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
|
||||
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
|
||||
compatibleWith: [setup.transcriber],
|
||||
considering: Self.captureFormat
|
||||
) else {
|
||||
throw ASRChunkError.formatUnsupported
|
||||
}
|
||||
analyzerFormat = format
|
||||
lock.withLock {
|
||||
chunkPreparedLocaleID = localeID
|
||||
chunkAnalyzerFormat = format
|
||||
}
|
||||
Self.debug(
|
||||
"chunk prepared analyzer format sr=\(Int(format.sampleRate)) " +
|
||||
"channels=\(format.channelCount) common=\(format.commonFormat.rawValue)"
|
||||
)
|
||||
}
|
||||
|
||||
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
|
||||
guard let pcm = Self.makeAnalyzerPCMBuffer(from: snapshot, format: analyzerFormat) else {
|
||||
throw ASRChunkError.formatUnsupported
|
||||
}
|
||||
|
||||
let analyzer = SpeechAnalyzer(modules: [setup.transcriber])
|
||||
try await analyzer.prepareToAnalyze(in: analyzerFormat)
|
||||
|
||||
let resultsTask = Task<String, Error> {
|
||||
var accumulator = ProgressiveDictationTranscriptAccumulator()
|
||||
for try await result in setup.transcriber.results {
|
||||
if Task.isCancelled { break }
|
||||
let text = String(result.text.characters)
|
||||
_ = accumulator.ingest(range: result.range, text: text)
|
||||
}
|
||||
return accumulator.finalize()
|
||||
}
|
||||
|
||||
let inputStream = AsyncStream<AnalyzerInput> { continuation in
|
||||
continuation.yield(AnalyzerInput(buffer: pcm))
|
||||
continuation.finish()
|
||||
}
|
||||
|
||||
let lastSampleTime = try await analyzer.analyzeSequence(inputStream)
|
||||
if let lastSampleTime {
|
||||
try await analyzer.finalizeAndFinish(through: lastSampleTime)
|
||||
} else {
|
||||
await analyzer.cancelAndFinishNow()
|
||||
}
|
||||
|
||||
return try await resultsTask.value
|
||||
}
|
||||
|
||||
private enum ASRChunkError: LocalizedError {
|
||||
case localeUnsupported
|
||||
case formatUnsupported
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .localeUnsupported:
|
||||
return SharedL10n.string("error.asr.localeUnsupported")
|
||||
case .formatUnsupported:
|
||||
return SharedL10n.string("error.asr.formatUnsupported")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical capture format: 16 kHz mono Float32 from `AudioCaptureService`
|
||||
/// / `PreviewASRController` before it reaches SpeechAnalyzer.
|
||||
private static let captureFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32,
|
||||
sampleRate: 16_000,
|
||||
channels: 1,
|
||||
interleaved: false
|
||||
)!
|
||||
|
||||
func transcribe(
|
||||
stream: AsyncStream<AudioBufferSnapshot>,
|
||||
locale: Locale
|
||||
) -> AsyncStream<ASREvent> {
|
||||
AsyncStream { continuation in
|
||||
continuation.yield(.capability(onDeviceSupported: true))
|
||||
|
||||
let task = Task { [weak self] in
|
||||
guard let self else { return }
|
||||
self.lock.withLock { self.analyzerFinished = false }
|
||||
defer {
|
||||
self.lock.withLock {
|
||||
self.analyzer = nil
|
||||
self.analyzerTask = nil
|
||||
self.analyzerFinished = true
|
||||
}
|
||||
}
|
||||
|
||||
do {
|
||||
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
|
||||
Self.debug("locale unsupported: \(locale.identifier(.bcp47))")
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.localeUnsupported")))
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
// Each pipelined chunk is ≤ 30 s; long dictation preset keeps a
|
||||
// single chunk coherent (Flow utterances run up to 3 min).
|
||||
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
|
||||
Self.debug(
|
||||
"stream setup locale=\(resolvedLocale.identifier(.bcp47)) " +
|
||||
"customLMEnabled=\(setup.customLanguageModelEnabled) " +
|
||||
"customLMAttached=\(setup.usesCustomLanguageModel) " +
|
||||
"clmState=\(Self.describeCLMState(setup.clmState))"
|
||||
)
|
||||
do {
|
||||
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
|
||||
} catch {
|
||||
Self.debug("asset prepare failed: \(error.localizedDescription)")
|
||||
FlowTrace.warn(
|
||||
"asr.local.stream.assetsNotReady",
|
||||
"locale=\(resolvedLocale.identifier(.bcp47)) "
|
||||
+ "error=\(error.localizedDescription)"
|
||||
)
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.assetsNotReady")))
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
|
||||
let newAnalyzer = SpeechAnalyzer(modules: [setup.transcriber])
|
||||
self.lock.withLock { self.analyzer = newAnalyzer }
|
||||
|
||||
guard let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(
|
||||
compatibleWith: [setup.transcriber],
|
||||
considering: Self.captureFormat
|
||||
) else {
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.formatUnsupported")))
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
|
||||
try await newAnalyzer.prepareToAnalyze(in: analyzerFormat)
|
||||
|
||||
let inputStream = self.makeInputStream(from: stream, analyzerFormat: analyzerFormat)
|
||||
|
||||
// Apple recommends consuming `transcriber.results` concurrently
|
||||
// while `analyzeSequence` drains the input stream.
|
||||
let resultsTask = Task<String, Error> {
|
||||
var accumulator = ProgressiveDictationTranscriptAccumulator()
|
||||
for try await result in setup.transcriber.results {
|
||||
if Task.isCancelled { break }
|
||||
let text = String(result.text.characters)
|
||||
guard let full = accumulator.ingest(range: result.range, text: text) else {
|
||||
continue
|
||||
}
|
||||
FlowTrace.transcript("asr.local.partial", full, "engine=local")
|
||||
continuation.yield(.partial(full))
|
||||
}
|
||||
return accumulator.finalize()
|
||||
}
|
||||
|
||||
let lastSampleTime = try await newAnalyzer.analyzeSequence(inputStream)
|
||||
|
||||
if let lastSampleTime {
|
||||
try await newAnalyzer.finalizeAndFinish(through: lastSampleTime)
|
||||
} else {
|
||||
await newAnalyzer.cancelAndFinishNow()
|
||||
}
|
||||
|
||||
let lastText: String
|
||||
do {
|
||||
lastText = try await resultsTask.value
|
||||
} catch {
|
||||
Self.debug("transcriber results failed: \(error.localizedDescription)")
|
||||
continuation.yield(.error(error.localizedDescription))
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
|
||||
let trimmed = lastText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if trimmed.isEmpty {
|
||||
FlowTrace.warn(
|
||||
"asr.local.stream.emptyFinal",
|
||||
"locale=\(resolvedLocale.identifier(.bcp47))"
|
||||
)
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
|
||||
} else {
|
||||
FlowTrace.transcript("asr.local.final", trimmed, "engine=local")
|
||||
continuation.yield(.final(trimmed))
|
||||
}
|
||||
continuation.finish()
|
||||
} catch is CancellationError {
|
||||
continuation.finish()
|
||||
} catch {
|
||||
Self.debug("SpeechAnalyzer failed: \(error.localizedDescription)")
|
||||
continuation.yield(.error(error.localizedDescription))
|
||||
continuation.finish()
|
||||
}
|
||||
}
|
||||
self.lock.withLock { self.analyzerTask = task }
|
||||
|
||||
continuation.onTermination = { @Sendable [weak self] _ in
|
||||
self?.cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private struct DiagnosticTranscriber {
|
||||
let transcriber: DictationTranscriber
|
||||
let customLanguageModelEnabled: Bool
|
||||
let usesCustomLanguageModel: Bool
|
||||
let clmState: CustomLanguageModelManager.PrepareState
|
||||
}
|
||||
|
||||
private static func makeDiagnosticTranscriber(locale: Locale) -> DiagnosticTranscriber {
|
||||
let defaults = AppGroup.defaultsIfAvailable
|
||||
let clmKey = AppGroupConfiguration.Keys.localASRCustomLanguageModelEnabled
|
||||
let clmEnabled = defaults?.object(forKey: clmKey) == nil
|
||||
? true
|
||||
: (defaults?.bool(forKey: clmKey) ?? true)
|
||||
let clmState = CustomLanguageModelManager.shared.currentState()
|
||||
let lmConfiguration = clmEnabled
|
||||
? CustomLanguageModelManager.shared.configurationForTranscription(locale: locale)
|
||||
: nil
|
||||
let transcriber = CustomLanguageModelManager.makeDictationTranscriber(
|
||||
locale: locale,
|
||||
lmConfiguration: lmConfiguration
|
||||
)
|
||||
return DiagnosticTranscriber(
|
||||
transcriber: transcriber,
|
||||
customLanguageModelEnabled: clmEnabled,
|
||||
usesCustomLanguageModel: lmConfiguration != nil,
|
||||
clmState: clmState
|
||||
)
|
||||
}
|
||||
|
||||
private static func describeCLMState(_ state: CustomLanguageModelManager.PrepareState) -> String {
|
||||
switch state {
|
||||
case .idle:
|
||||
return "idle"
|
||||
case .preparing:
|
||||
return "preparing"
|
||||
case .ready:
|
||||
return "ready"
|
||||
case .failed(let message):
|
||||
return "failed(\(message))"
|
||||
}
|
||||
}
|
||||
|
||||
private static func rms(of samples: [Float]) -> Float {
|
||||
guard !samples.isEmpty else { return 0 }
|
||||
var sum: Float = 0
|
||||
for sample in samples {
|
||||
sum += sample * sample
|
||||
}
|
||||
return sqrtf(sum / Float(samples.count))
|
||||
}
|
||||
|
||||
private static func elapsed(_ start: Date) -> String {
|
||||
String(format: "%.2f", Date().timeIntervalSince(start))
|
||||
}
|
||||
|
||||
private static func debug(_ message: String) {
|
||||
OSGLog.asr.info("\(message, privacy: .public)")
|
||||
}
|
||||
|
||||
private static func prepareAssetsIfNeeded(
|
||||
for transcriber: DictationTranscriber,
|
||||
locale: Locale
|
||||
) async throws {
|
||||
let localeID = locale.identifier(.bcp47)
|
||||
let startedAt = Date()
|
||||
do {
|
||||
_ = try await AssetInventory.reserve(locale: locale)
|
||||
Self.debug("asset reserve ok locale=\(localeID)")
|
||||
} catch {
|
||||
// Reservation may already exist or slots are full; continue, but
|
||||
// log it so local-ASR setup failures are not hidden behind a later
|
||||
// "no speech" timeout.
|
||||
Self.debug("asset reserve non-fatal locale=\(localeID) error=\(error.localizedDescription)")
|
||||
}
|
||||
do {
|
||||
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
|
||||
Self.debug("asset install required locale=\(localeID)")
|
||||
try await request.downloadAndInstall()
|
||||
Self.debug("asset install done locale=\(localeID) elapsed=\(elapsed(startedAt))s")
|
||||
} else {
|
||||
Self.debug("asset already installed locale=\(localeID) elapsed=\(elapsed(startedAt))s")
|
||||
}
|
||||
} catch {
|
||||
Self.debug("asset prepare failed locale=\(localeID) elapsed=\(elapsed(startedAt))s error=\(error.localizedDescription)")
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
let (task, currentAnalyzer, finished) = lock.withLock { () -> (Task<Void, Never>?, SpeechAnalyzer?, Bool) in
|
||||
let t = analyzerTask
|
||||
let a = analyzer
|
||||
let f = analyzerFinished
|
||||
analyzerTask = nil
|
||||
analyzer = nil
|
||||
return (t, a, f)
|
||||
}
|
||||
task?.cancel()
|
||||
guard !finished, let currentAnalyzer else { return }
|
||||
Task { await currentAnalyzer.cancelAndFinishNow() }
|
||||
}
|
||||
|
||||
/// Maps 16 kHz Float32 snapshots into `AnalyzerInput` using the format
|
||||
/// returned by `bestAvailableAudioFormat(compatibleWith:considering:)`.
|
||||
private func makeInputStream(
|
||||
from stream: AsyncStream<AudioBufferSnapshot>,
|
||||
analyzerFormat: AVAudioFormat
|
||||
) -> AsyncStream<AnalyzerInput> {
|
||||
AsyncStream { continuation in
|
||||
Task {
|
||||
for await snap in stream {
|
||||
guard !snap.samples.isEmpty else { continue }
|
||||
guard let pcm = Self.makeAnalyzerPCMBuffer(from: snap, format: analyzerFormat) else {
|
||||
continue
|
||||
}
|
||||
continuation.yield(AnalyzerInput(buffer: pcm))
|
||||
}
|
||||
continuation.finish()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static func makeAnalyzerPCMBuffer(
|
||||
from snap: AudioBufferSnapshot,
|
||||
format: AVAudioFormat
|
||||
) -> AVAudioPCMBuffer? {
|
||||
let capacity = AVAudioFrameCount(snap.samples.count)
|
||||
guard capacity > 0,
|
||||
let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else {
|
||||
return nil
|
||||
}
|
||||
pcm.frameLength = capacity
|
||||
|
||||
switch format.commonFormat {
|
||||
case .pcmFormatInt16:
|
||||
guard let dst = pcm.int16ChannelData?[0] else { return nil }
|
||||
snap.samples.withUnsafeBufferPointer { src in
|
||||
ASRServiceFactory.convertFloat32ToInt16(
|
||||
source: src.baseAddress,
|
||||
sourceCount: src.count,
|
||||
destination: dst
|
||||
)
|
||||
}
|
||||
case .pcmFormatFloat32:
|
||||
guard let dst = pcm.floatChannelData?[0] else { return nil }
|
||||
snap.samples.withUnsafeBufferPointer { src in
|
||||
guard let base = src.baseAddress else { return }
|
||||
memcpy(dst, base, src.count * MemoryLayout<Float>.stride)
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return pcm
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
// ChunkedUtterancePipeline.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Pipelined Flow utterance ASR: split PCM while recording, transcribe chunks
|
||||
// serially on a background queue, stitch partials for display and delivery.
|
||||
|
||||
import Foundation
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public struct ChunkedUtteranceSuccess: Sendable, Equatable {
|
||||
public let text: String
|
||||
/// Same transcript with internal pause markers, used only by polish.
|
||||
public let textWithPauseMarks: String
|
||||
/// Non-fatal per-chunk ASR issues (delivered as soft warning when non-empty).
|
||||
public let chunkWarnings: [String]
|
||||
|
||||
public init(
|
||||
text: String,
|
||||
textWithPauseMarks: String? = nil,
|
||||
chunkWarnings: [String] = []
|
||||
) {
|
||||
self.text = text
|
||||
self.textWithPauseMarks = textWithPauseMarks ?? text
|
||||
self.chunkWarnings = chunkWarnings
|
||||
}
|
||||
}
|
||||
|
||||
public enum ChunkedUtterancePipelineOutcome: Sendable, Equatable {
|
||||
case success(ChunkedUtteranceSuccess)
|
||||
case failure(String)
|
||||
case cancelled
|
||||
}
|
||||
|
||||
/// Thread-safe queue between the chunk feeder and ASR worker.
|
||||
private actor ChunkWorkQueue {
|
||||
private var items: [UtteranceAudioChunk] = []
|
||||
private var finished = false
|
||||
private var waiters: [CheckedContinuation<UtteranceAudioChunk?, Never>] = []
|
||||
|
||||
func enqueue(_ chunk: UtteranceAudioChunk) {
|
||||
items.append(chunk)
|
||||
resumeWaiters()
|
||||
}
|
||||
|
||||
func markFinished() {
|
||||
finished = true
|
||||
resumeWaiters()
|
||||
}
|
||||
|
||||
func dequeue() async -> UtteranceAudioChunk? {
|
||||
if !items.isEmpty {
|
||||
return items.removeFirst()
|
||||
}
|
||||
if finished {
|
||||
return nil
|
||||
}
|
||||
return await withCheckedContinuation { continuation in
|
||||
waiters.append(continuation)
|
||||
}
|
||||
}
|
||||
|
||||
private func resumeWaiters() {
|
||||
while !waiters.isEmpty {
|
||||
if !items.isEmpty {
|
||||
let waiter = waiters.removeFirst()
|
||||
waiter.resume(returning: items.removeFirst())
|
||||
} else if finished {
|
||||
let waiter = waiters.removeFirst()
|
||||
waiter.resume(returning: nil)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public actor ChunkedUtterancePipeline {
|
||||
private let asr: any ASRChunkTranscribing
|
||||
private let locale: Locale
|
||||
private let config: FlowUtteranceChunkConfig
|
||||
private var cancelled = false
|
||||
|
||||
public init(
|
||||
asr: any ASRChunkTranscribing,
|
||||
locale: Locale,
|
||||
config: FlowUtteranceChunkConfig = .flowDefault
|
||||
) {
|
||||
self.asr = asr
|
||||
self.locale = locale
|
||||
self.config = config
|
||||
}
|
||||
|
||||
public func cancel() {
|
||||
cancelled = true
|
||||
asr.cancel()
|
||||
}
|
||||
|
||||
/// Consume `stream` until finished; ASR runs off the caller's actor while recording continues.
|
||||
public func transcribe(
|
||||
stream: AsyncStream<AudioBufferSnapshot>,
|
||||
onPartial: @Sendable @escaping (String) -> Void
|
||||
) async -> ChunkedUtterancePipelineOutcome {
|
||||
asr.resetForNewUtterance()
|
||||
|
||||
let queue = ChunkWorkQueue()
|
||||
var stitcher = UtteranceTranscriptStitcher()
|
||||
var chunkWarnings: [String] = []
|
||||
var failedChunks = 0
|
||||
var processedChunks = 0
|
||||
var previousChunkSamples: [Float] = []
|
||||
var lastChunkSamples = 0
|
||||
var didRetryEmptyFinal = false
|
||||
|
||||
let feeder = Task {
|
||||
for await chunk in UtteranceStreamChunker.chunks(from: stream, config: config) {
|
||||
if Task.isCancelled { break }
|
||||
await queue.enqueue(chunk)
|
||||
}
|
||||
await queue.markFinished()
|
||||
}
|
||||
|
||||
while true {
|
||||
if cancelled || Task.isCancelled {
|
||||
feeder.cancel()
|
||||
return .cancelled
|
||||
}
|
||||
|
||||
guard let chunk = await queue.dequeue() else { break }
|
||||
|
||||
if chunk.isLast && chunk.samples.isEmpty {
|
||||
continue
|
||||
}
|
||||
|
||||
processedChunks += 1
|
||||
lastChunkSamples = chunk.samples.count
|
||||
|
||||
if let preMerge = FinalChunkRecovery.preMergePlan(
|
||||
chunk: chunk,
|
||||
processedChunks: processedChunks,
|
||||
previousChunkSamples: previousChunkSamples,
|
||||
config: config
|
||||
) {
|
||||
FlowPipelineDiagnostics.logFinalChunkRecovery(
|
||||
action: "preMerge",
|
||||
chunkIndex: chunk.index
|
||||
)
|
||||
let mergedResult = await transcribeChunkWithRetry(
|
||||
samples: preMerge.samples,
|
||||
chunkIndex: chunk.index
|
||||
)
|
||||
switch mergedResult {
|
||||
case .success(let text):
|
||||
// Empty / whitespace merge must NOT wipe a prior good segment
|
||||
// (`append` ignores empty text, so remove-then-append would
|
||||
// silently drop the only transcript — the AC327-style bug).
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if trimmed.isEmpty {
|
||||
FlowPipelineDiagnostics.logFinalChunkRecovery(
|
||||
action: "preMergeKeepPrior",
|
||||
chunkIndex: chunk.index
|
||||
)
|
||||
} else {
|
||||
stitcher.removeLastSegment()
|
||||
stitcher.append(
|
||||
index: preMerge.stitchIndex,
|
||||
text: text,
|
||||
trailingPauseSeconds: chunk.trailingPauseSeconds
|
||||
)
|
||||
publishPartial(from: stitcher, onPartial: onPartial)
|
||||
}
|
||||
case .failure(let message):
|
||||
// Keep prior stitcher text; treat as a soft chunk warning.
|
||||
failedChunks += 1
|
||||
chunkWarnings.append(
|
||||
SharedL10n.format(
|
||||
"error.asr.chunkFailed",
|
||||
chunk.index + 1,
|
||||
message
|
||||
)
|
||||
)
|
||||
case .cancelled:
|
||||
feeder.cancel()
|
||||
return .cancelled
|
||||
}
|
||||
previousChunkSamples = chunk.samples
|
||||
continue
|
||||
}
|
||||
|
||||
let result = await transcribeChunkWithRetry(
|
||||
samples: chunk.samples,
|
||||
chunkIndex: chunk.index
|
||||
)
|
||||
logChunkOutcome(chunk: chunk, result: result)
|
||||
switch result {
|
||||
case .success(let text):
|
||||
if chunk.isLast,
|
||||
!didRetryEmptyFinal,
|
||||
let retry = FinalChunkRecovery.emptyResultRetryPlan(
|
||||
chunk: chunk,
|
||||
previousChunkSamples: previousChunkSamples,
|
||||
config: config,
|
||||
asrText: text
|
||||
) {
|
||||
didRetryEmptyFinal = true
|
||||
FlowPipelineDiagnostics.logFinalChunkRecovery(
|
||||
action: "emptyRetry",
|
||||
chunkIndex: chunk.index
|
||||
)
|
||||
let retryResult = await transcribeChunkWithRetry(
|
||||
samples: retry.samples,
|
||||
chunkIndex: chunk.index
|
||||
)
|
||||
switch retryResult {
|
||||
case .success(let retryText):
|
||||
let trimmed = retryText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !trimmed.isEmpty {
|
||||
if retry.stitchIndex < chunk.index {
|
||||
stitcher.removeLastSegment()
|
||||
}
|
||||
stitcher.append(
|
||||
index: retry.stitchIndex,
|
||||
text: retryText,
|
||||
trailingPauseSeconds: chunk.trailingPauseSeconds
|
||||
)
|
||||
publishPartial(from: stitcher, onPartial: onPartial)
|
||||
} else {
|
||||
stitcher.append(
|
||||
index: chunk.index,
|
||||
text: text,
|
||||
trailingPauseSeconds: chunk.trailingPauseSeconds
|
||||
)
|
||||
publishPartial(from: stitcher, onPartial: onPartial)
|
||||
}
|
||||
case .failure(let message):
|
||||
stitcher.append(
|
||||
index: chunk.index,
|
||||
text: text,
|
||||
trailingPauseSeconds: chunk.trailingPauseSeconds
|
||||
)
|
||||
publishPartial(from: stitcher, onPartial: onPartial)
|
||||
failedChunks += 1
|
||||
chunkWarnings.append(
|
||||
SharedL10n.format(
|
||||
"error.asr.chunkFailed",
|
||||
chunk.index + 1,
|
||||
message
|
||||
)
|
||||
)
|
||||
case .cancelled:
|
||||
feeder.cancel()
|
||||
return .cancelled
|
||||
}
|
||||
} else {
|
||||
stitcher.append(
|
||||
index: chunk.index,
|
||||
text: text,
|
||||
trailingPauseSeconds: chunk.trailingPauseSeconds
|
||||
)
|
||||
publishPartial(from: stitcher, onPartial: onPartial)
|
||||
}
|
||||
case .failure(let message):
|
||||
failedChunks += 1
|
||||
chunkWarnings.append(
|
||||
SharedL10n.format(
|
||||
"error.asr.chunkFailed",
|
||||
chunk.index + 1,
|
||||
message
|
||||
)
|
||||
)
|
||||
case .cancelled:
|
||||
feeder.cancel()
|
||||
return .cancelled
|
||||
}
|
||||
|
||||
previousChunkSamples = chunk.samples
|
||||
}
|
||||
|
||||
_ = await feeder.value
|
||||
|
||||
let finalText = stitcher.composedSafely().trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let markedText = stitcher.composedWithPauseMarks()
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
FlowPipelineDiagnostics.logChunkFinalize(
|
||||
chunkCount: processedChunks,
|
||||
lastChunkSamples: lastChunkSamples,
|
||||
stitchedLength: finalText.count,
|
||||
chunkWarnings: chunkWarnings.count
|
||||
)
|
||||
|
||||
if finalText.isEmpty {
|
||||
FlowTrace.warn(
|
||||
"pipeline.stitch.empty",
|
||||
"chunks=\(processedChunks) failedChunks=\(failedChunks) "
|
||||
+ "lastChunkSamples=\(lastChunkSamples) warnings=\(chunkWarnings.count)"
|
||||
)
|
||||
if failedChunks > 0, processedChunks == failedChunks {
|
||||
return .failure(SharedL10n.string("error.asr.noSpeech"))
|
||||
}
|
||||
return .failure(SharedL10n.string("error.asr.noSpeech"))
|
||||
}
|
||||
|
||||
FlowTrace.transcript(
|
||||
"asr.stitched",
|
||||
finalText,
|
||||
"chunks=\(processedChunks) failedChunks=\(failedChunks) warnings=\(chunkWarnings.count)"
|
||||
)
|
||||
return .success(
|
||||
ChunkedUtteranceSuccess(
|
||||
text: finalText,
|
||||
textWithPauseMarks: markedText,
|
||||
chunkWarnings: chunkWarnings
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private func transcribeChunk(samples: [Float]) async -> ASRChunkResult {
|
||||
let asr = self.asr
|
||||
let locale = self.locale
|
||||
return await Task.detached(priority: .userInitiated) {
|
||||
await asr.transcribeChunk(samples: samples, locale: locale)
|
||||
}.value
|
||||
}
|
||||
|
||||
/// Retry one failed chunk before advancing the serial worker. Keeping the
|
||||
/// same PCM samples prevents a transient request failure from creating an
|
||||
/// undetectable hole in an otherwise fluent stitched transcript.
|
||||
private func transcribeChunkWithRetry(
|
||||
samples: [Float],
|
||||
chunkIndex: Int
|
||||
) async -> ASRChunkResult {
|
||||
let first = await transcribeChunk(samples: samples)
|
||||
guard case .failure(let message) = first else { return first }
|
||||
guard !cancelled, !Task.isCancelled else { return .cancelled }
|
||||
|
||||
FlowTrace.warn(
|
||||
"pipeline.chunk.retry",
|
||||
"chunk=\(chunkIndex) samples=\(samples.count) error=\(message)"
|
||||
)
|
||||
do {
|
||||
try await Task.sleep(nanoseconds: 150_000_000)
|
||||
} catch {
|
||||
return .cancelled
|
||||
}
|
||||
guard !cancelled, !Task.isCancelled else { return .cancelled }
|
||||
return await transcribeChunk(samples: samples)
|
||||
}
|
||||
|
||||
/// Pairs each chunk's audio with the text it produced, so an empty
|
||||
/// transcript can be attributed to either silent audio or a mute engine.
|
||||
private func logChunkOutcome(chunk: UtteranceAudioChunk, result: ASRChunkResult) {
|
||||
let audio = "chunk=\(chunk.index) samples=\(chunk.samples.count) "
|
||||
+ "seconds=\(FlowTrace.seconds(samples: chunk.samples.count, sampleRate: config.sampleRate)) "
|
||||
+ "rms=\(FlowTrace.rms(chunk.samples)) isLast=\(chunk.isLast ? 1 : 0)"
|
||||
switch result {
|
||||
case .success(let text):
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if trimmed.isEmpty {
|
||||
FlowTrace.warn("pipeline.chunk.emptyText", audio)
|
||||
} else {
|
||||
FlowTrace.transcript("asr.chunk", trimmed, audio)
|
||||
}
|
||||
case .failure(let message):
|
||||
FlowTrace.warn("pipeline.chunk.failed", "\(audio) error=\(message)")
|
||||
case .cancelled:
|
||||
FlowTrace.pipeline("chunk.cancelled", audio)
|
||||
}
|
||||
}
|
||||
|
||||
private func publishPartial(
|
||||
from stitcher: UtteranceTranscriptStitcher,
|
||||
onPartial: @Sendable (String) -> Void
|
||||
) {
|
||||
let partial = stitcher.composedSafely()
|
||||
if !partial.isEmpty {
|
||||
onPartial(partial)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
// AlibabaVocabularySync.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Syncs PersonalDictionary → DashScope custom vocabulary (Fun-ASR Flash).
|
||||
|
||||
import Foundation
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public enum AlibabaVocabularySync {
|
||||
public enum Keys {
|
||||
public static let vocabularyId = "config.alibabaASRVocabularyId"
|
||||
public static let fingerprint = "config.alibabaASRVocabularyFingerprint"
|
||||
}
|
||||
|
||||
private static let vocabularyPrefix = "osgkb"
|
||||
|
||||
/// Returns a ready `vocabulary_id`, creating or updating the remote list as needed.
|
||||
public static func ensureVocabularyID(
|
||||
dictionary: PersonalDictionary,
|
||||
apiKey: String,
|
||||
targetModel: String = CloudASRModelCatalog.alibabaVocabularyTargetModel,
|
||||
defaults: UserDefaults,
|
||||
session: URLSession = .shared
|
||||
) async throws -> String? {
|
||||
let entries = dictionary.alibabaHotwordEntries()
|
||||
guard !entries.isEmpty else {
|
||||
clearCache(defaults: defaults)
|
||||
return nil
|
||||
}
|
||||
|
||||
let fingerprint = dictionary.vocabularySyncFingerprint()
|
||||
if let cachedID = defaults.string(forKey: Keys.vocabularyId),
|
||||
defaults.string(forKey: Keys.fingerprint) == fingerprint,
|
||||
!cachedID.isEmpty {
|
||||
return cachedID
|
||||
}
|
||||
|
||||
let url = try customizationURL()
|
||||
if let existingID = defaults.string(forKey: Keys.vocabularyId), !existingID.isEmpty {
|
||||
try await updateVocabulary(
|
||||
id: existingID,
|
||||
entries: entries,
|
||||
apiKey: apiKey,
|
||||
url: url,
|
||||
session: session
|
||||
)
|
||||
cache(id: existingID, fingerprint: fingerprint, defaults: defaults)
|
||||
return existingID
|
||||
}
|
||||
|
||||
let createdID = try await createVocabulary(
|
||||
entries: entries,
|
||||
targetModel: targetModel,
|
||||
apiKey: apiKey,
|
||||
url: url,
|
||||
session: session
|
||||
)
|
||||
cache(id: createdID, fingerprint: fingerprint, defaults: defaults)
|
||||
return createdID
|
||||
}
|
||||
|
||||
public static func clearCache(defaults: UserDefaults) {
|
||||
defaults.removeObject(forKey: Keys.vocabularyId)
|
||||
defaults.removeObject(forKey: Keys.fingerprint)
|
||||
}
|
||||
|
||||
private static func cache(id: String, fingerprint: String, defaults: UserDefaults) {
|
||||
defaults.set(id, forKey: Keys.vocabularyId)
|
||||
defaults.set(fingerprint, forKey: Keys.fingerprint)
|
||||
}
|
||||
|
||||
private static func customizationURL() throws -> URL {
|
||||
let raw = CloudASRModelCatalog.alibabaAPIBase + CloudASRModelCatalog.alibabaCustomizationPath
|
||||
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
|
||||
return url
|
||||
}
|
||||
|
||||
private static func createVocabulary(
|
||||
entries: [AlibabaHotwordEntry],
|
||||
targetModel: String,
|
||||
apiKey: String,
|
||||
url: URL,
|
||||
session: URLSession
|
||||
) async throws -> String {
|
||||
let vocabulary = entries.map { entry -> [String: Any] in
|
||||
var item: [String: Any] = ["text": entry.text, "weight": entry.weight]
|
||||
if let lang = entry.lang { item["lang"] = lang }
|
||||
return item
|
||||
}
|
||||
let body: [String: Any] = [
|
||||
"model": "speech-biasing",
|
||||
"input": [
|
||||
"action": "create_vocabulary",
|
||||
"target_model": targetModel,
|
||||
"prefix": vocabularyPrefix,
|
||||
"vocabulary": vocabulary,
|
||||
] as [String: Any],
|
||||
]
|
||||
let data = try await postJSON(body, to: url, apiKey: apiKey, session: session)
|
||||
guard let id = parseVocabularyID(from: data) else {
|
||||
throw CloudASRError.decoding("missing vocabulary_id")
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
private static func updateVocabulary(
|
||||
id: String,
|
||||
entries: [AlibabaHotwordEntry],
|
||||
apiKey: String,
|
||||
url: URL,
|
||||
session: URLSession
|
||||
) async throws {
|
||||
let vocabulary = entries.map { entry -> [String: Any] in
|
||||
var item: [String: Any] = ["text": entry.text, "weight": entry.weight]
|
||||
if let lang = entry.lang { item["lang"] = lang }
|
||||
return item
|
||||
}
|
||||
let body: [String: Any] = [
|
||||
"model": "speech-biasing",
|
||||
"input": [
|
||||
"action": "update_vocabulary",
|
||||
"vocabulary_id": id,
|
||||
"vocabulary": vocabulary,
|
||||
] as [String: Any],
|
||||
]
|
||||
_ = try await postJSON(body, to: url, apiKey: apiKey, session: session)
|
||||
}
|
||||
|
||||
private static func postJSON(
|
||||
_ body: [String: Any],
|
||||
to url: URL,
|
||||
apiKey: String,
|
||||
session: URLSession
|
||||
) async throws -> [String: Any] {
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
||||
request.httpBody = try JSONSerialization.data(withJSONObject: body)
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
guard let http = response as? HTTPURLResponse else {
|
||||
throw CloudASRError.transport("non-HTTP response")
|
||||
}
|
||||
let json = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any]
|
||||
guard (200..<300).contains(http.statusCode) else {
|
||||
let message = parseAPIErrorMessage(from: json)
|
||||
throw CloudASRError.http(status: http.statusCode, message: message)
|
||||
}
|
||||
return json ?? [:]
|
||||
}
|
||||
|
||||
private static func parseVocabularyID(from json: [String: Any]) -> String? {
|
||||
if let output = json["output"] as? [String: Any],
|
||||
let id = output["vocabulary_id"] as? String {
|
||||
return id
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
private static func parseAPIErrorMessage(from json: [String: Any]?) -> String? {
|
||||
guard let json else { return nil }
|
||||
if let message = json["message"] as? String { return message }
|
||||
if let error = json["error"] as? [String: Any],
|
||||
let message = error["message"] as? String {
|
||||
return message
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
// BailianRealtimeASRClient.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Alibaba Cloud Bailian / DashScope realtime ASR over the classic inference
|
||||
// WebSocket (`/api-ws/v1/inference`). Utterance-level duplex session with
|
||||
// interim `result-generated` partials; batch `transcribe(samples:)` remains
|
||||
// for connection probes and chunk fallback.
|
||||
|
||||
import Foundation
|
||||
import os
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
struct BailianRealtimeASRClient: CloudASRTranscribing, CloudASRStreamingCapable {
|
||||
let apiKey: String
|
||||
let endpoint: String
|
||||
let model: String
|
||||
let vocabularyID: String?
|
||||
let session: URLSession
|
||||
|
||||
/// 100 ms of 16 kHz / 16-bit / mono PCM.
|
||||
static let targetChunkBytes = 3_200
|
||||
static let startTimeout: TimeInterval = 8
|
||||
static let finalTimeout: TimeInterval = 12
|
||||
private static let sessionTimeout: TimeInterval = startTimeout + finalTimeout + 4
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func openStreamingSession(
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary,
|
||||
onPartial: @escaping @Sendable (String) -> Void
|
||||
) async throws -> any CloudASRStreamingSession {
|
||||
_ = locale
|
||||
_ = dictionary
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
let url = try resolvedEndpointURL()
|
||||
let resolvedModel = model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? CloudASRModelCatalog.alibabaFunASRRealtime
|
||||
: model.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.timeoutInterval = 8
|
||||
request.setValue(
|
||||
"bearer \(apiKey.trimmingCharacters(in: .whitespacesAndNewlines))",
|
||||
forHTTPHeaderField: "Authorization"
|
||||
)
|
||||
|
||||
let wsTask = session.webSocketTask(with: request)
|
||||
wsTask.resume()
|
||||
let live = BailianStreamingSession(
|
||||
wsTask: wsTask,
|
||||
model: resolvedModel,
|
||||
vocabularyID: vocabularyID,
|
||||
onPartial: onPartial
|
||||
)
|
||||
try await live.start()
|
||||
return live
|
||||
}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
guard sampleRate == 16_000 else {
|
||||
throw CloudASRError.transport("Bailian realtime expects 16 kHz audio")
|
||||
}
|
||||
guard !samples.isEmpty else { throw CloudASRError.emptyTranscript }
|
||||
|
||||
let session = try await openStreamingSession(
|
||||
locale: locale,
|
||||
dictionary: dictionary,
|
||||
onPartial: { _ in }
|
||||
)
|
||||
try await session.append(samples: samples)
|
||||
let text = try await session.finish()
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { throw CloudASRError.emptyTranscript }
|
||||
return trimmed
|
||||
}
|
||||
|
||||
/// Settings connection probe: handshake to `task-started` only.
|
||||
func probeConnection() async throws {
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
|
||||
let url = try resolvedEndpointURL()
|
||||
let taskID = UUID().uuidString.replacingOccurrences(of: "-", with: "")
|
||||
let resolvedModel = model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? CloudASRModelCatalog.alibabaFunASRRealtime
|
||||
: model.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.timeoutInterval = 8
|
||||
request.setValue(
|
||||
"bearer \(apiKey.trimmingCharacters(in: .whitespacesAndNewlines))",
|
||||
forHTTPHeaderField: "Authorization"
|
||||
)
|
||||
|
||||
let wsTask = session.webSocketTask(with: request)
|
||||
wsTask.resume()
|
||||
|
||||
try await withThrowingTaskGroup(of: Void.self) { group in
|
||||
let events = BailianEventStream(task: wsTask, onPartial: nil)
|
||||
|
||||
group.addTask {
|
||||
defer { events.cancel() }
|
||||
try await BailianRealtimeASRClient.sendText(
|
||||
BailianRealtimeASRClient.runTaskMessage(
|
||||
taskID: taskID,
|
||||
model: resolvedModel,
|
||||
vocabularyID: nil
|
||||
),
|
||||
task: wsTask
|
||||
)
|
||||
try await events.waitForStarted(timeout: Self.startTimeout)
|
||||
try? await BailianRealtimeASRClient.sendText(
|
||||
BailianRealtimeASRClient.finishTaskMessage(taskID: taskID),
|
||||
task: wsTask
|
||||
)
|
||||
}
|
||||
|
||||
group.addTask {
|
||||
try await Task.sleep(nanoseconds: UInt64(Self.startTimeout * 1_000_000_000))
|
||||
events.cancel()
|
||||
wsTask.cancel(with: .goingAway, reason: nil)
|
||||
throw CloudASRError.transport("connection probe timed out")
|
||||
}
|
||||
|
||||
_ = try await group.next()
|
||||
group.cancelAll()
|
||||
}
|
||||
}
|
||||
|
||||
private func resolvedEndpointURL() throws -> URL {
|
||||
let raw = endpoint.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? CloudASRModelCatalog.bailianDefaultEndpoint
|
||||
: endpoint.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
|
||||
return url
|
||||
}
|
||||
|
||||
static func sendText(_ text: String, task: URLSessionWebSocketTask) async throws {
|
||||
do {
|
||||
try await task.send(.string(text))
|
||||
} catch {
|
||||
throw CloudASRError.transport(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
static func sendBinary(_ data: Data, task: URLSessionWebSocketTask) async throws {
|
||||
do {
|
||||
try await task.send(.data(data))
|
||||
} catch {
|
||||
throw CloudASRError.transport(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
/// Overlap-aware join to avoid cumulative duplicate text from interim replays.
|
||||
static func mergeSegments(_ segments: [String]) -> String {
|
||||
var result = ""
|
||||
for segment in segments {
|
||||
if result.isEmpty {
|
||||
result = segment
|
||||
continue
|
||||
}
|
||||
let resultChars = Array(result)
|
||||
let segmentChars = Array(segment)
|
||||
let maxOverlap = min(resultChars.count, segmentChars.count)
|
||||
var overlap = 0
|
||||
if maxOverlap >= 2 {
|
||||
for length in stride(from: maxOverlap, through: 2, by: -1) {
|
||||
let tail = resultChars.suffix(length)
|
||||
let head = segmentChars.prefix(length)
|
||||
if tail.elementsEqual(head) {
|
||||
overlap = length
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
result.append(contentsOf: segmentChars.dropFirst(overlap))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
static func runTaskMessage(taskID: String, model: String, vocabularyID: String?) -> String {
|
||||
var parameters: [String: Any] = [
|
||||
"sample_rate": 16_000,
|
||||
"format": "pcm",
|
||||
]
|
||||
if let vocabularyID = vocabularyID?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!vocabularyID.isEmpty {
|
||||
parameters["vocabulary_id"] = vocabularyID
|
||||
}
|
||||
let body: [String: Any] = [
|
||||
"header": [
|
||||
"action": "run-task",
|
||||
"task_id": taskID,
|
||||
"streaming": "duplex",
|
||||
],
|
||||
"payload": [
|
||||
"task_group": "audio",
|
||||
"task": "asr",
|
||||
"function": "recognition",
|
||||
"model": model,
|
||||
"parameters": parameters,
|
||||
"input": [:] as [String: Any],
|
||||
],
|
||||
]
|
||||
guard let data = try? JSONSerialization.data(withJSONObject: body),
|
||||
let json = String(data: data, encoding: .utf8) else {
|
||||
return "{}"
|
||||
}
|
||||
return json
|
||||
}
|
||||
|
||||
static func finishTaskMessage(taskID: String) -> String {
|
||||
let body: [String: Any] = [
|
||||
"header": [
|
||||
"action": "finish-task",
|
||||
"task_id": taskID,
|
||||
"streaming": "duplex",
|
||||
],
|
||||
"payload": ["input": [:] as [String: Any]],
|
||||
]
|
||||
guard let data = try? JSONSerialization.data(withJSONObject: body),
|
||||
let json = String(data: data, encoding: .utf8) else {
|
||||
return "{}"
|
||||
}
|
||||
return json
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Utterance session
|
||||
|
||||
private final class BailianStreamingSession: CloudASRStreamingSession, @unchecked Sendable {
|
||||
private let wsTask: URLSessionWebSocketTask
|
||||
private let model: String
|
||||
private let vocabularyID: String?
|
||||
private let onPartial: @Sendable (String) -> Void
|
||||
private let events: BailianEventStream
|
||||
private let taskID: String
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
private var started = false
|
||||
private var pcmBuffer = Data()
|
||||
|
||||
init(
|
||||
wsTask: URLSessionWebSocketTask,
|
||||
model: String,
|
||||
vocabularyID: String?,
|
||||
onPartial: @escaping @Sendable (String) -> Void
|
||||
) {
|
||||
self.wsTask = wsTask
|
||||
self.model = model
|
||||
self.vocabularyID = vocabularyID
|
||||
self.onPartial = onPartial
|
||||
self.taskID = UUID().uuidString.replacingOccurrences(of: "-", with: "")
|
||||
self.events = BailianEventStream(task: wsTask, onPartial: onPartial)
|
||||
}
|
||||
|
||||
func start() async throws {
|
||||
try await BailianRealtimeASRClient.sendText(
|
||||
BailianRealtimeASRClient.runTaskMessage(
|
||||
taskID: taskID,
|
||||
model: model,
|
||||
vocabularyID: vocabularyID
|
||||
),
|
||||
task: wsTask
|
||||
)
|
||||
try await events.waitForStarted(timeout: BailianRealtimeASRClient.startTimeout)
|
||||
lock.withLock { started = true }
|
||||
}
|
||||
|
||||
func append(samples: [Float]) async throws {
|
||||
guard lock.withLock({ started }) else {
|
||||
throw CloudASRError.transport("Bailian session not started")
|
||||
}
|
||||
let pcm = CloudASRStreamingPCM.pcm16LE(samples: samples)
|
||||
let frames: [Data] = lock.withLock {
|
||||
pcmBuffer.append(pcm)
|
||||
var frames: [Data] = []
|
||||
while pcmBuffer.count >= BailianRealtimeASRClient.targetChunkBytes {
|
||||
let frame = pcmBuffer.prefix(BailianRealtimeASRClient.targetChunkBytes)
|
||||
frames.append(Data(frame))
|
||||
pcmBuffer.removeFirst(BailianRealtimeASRClient.targetChunkBytes)
|
||||
}
|
||||
return frames
|
||||
}
|
||||
for frame in frames {
|
||||
try await BailianRealtimeASRClient.sendBinary(frame, task: wsTask)
|
||||
}
|
||||
}
|
||||
|
||||
func finish() async throws -> String {
|
||||
// Flush remaining PCM (pad short last frame as-is — server tolerates).
|
||||
let trailing: Data = lock.withLock {
|
||||
let data = pcmBuffer
|
||||
pcmBuffer.removeAll(keepingCapacity: false)
|
||||
return data
|
||||
}
|
||||
if !trailing.isEmpty {
|
||||
try await BailianRealtimeASRClient.sendBinary(trailing, task: wsTask)
|
||||
}
|
||||
// Avoid emptyAudio race on very short clips.
|
||||
try? await Task.sleep(nanoseconds: 120_000_000)
|
||||
try await BailianRealtimeASRClient.sendText(
|
||||
BailianRealtimeASRClient.finishTaskMessage(taskID: taskID),
|
||||
task: wsTask
|
||||
)
|
||||
return try await events.waitForFinalText(timeout: BailianRealtimeASRClient.finalTimeout)
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
events.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Concurrent read loop
|
||||
|
||||
private final class BailianEventStream: @unchecked Sendable {
|
||||
private let task: URLSessionWebSocketTask
|
||||
private let onPartial: (@Sendable (String) -> Void)?
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
private var started = false
|
||||
private var finalText: String?
|
||||
private var failure: Error?
|
||||
private var readTask: Task<Void, Never>?
|
||||
|
||||
init(task: URLSessionWebSocketTask, onPartial: (@Sendable (String) -> Void)?) {
|
||||
self.task = task
|
||||
self.onPartial = onPartial
|
||||
readTask = Task { [weak self] in
|
||||
await self?.readLoop()
|
||||
}
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
readTask?.cancel()
|
||||
task.cancel(with: .goingAway, reason: nil)
|
||||
}
|
||||
|
||||
func waitForStarted(timeout: TimeInterval) async throws {
|
||||
let deadline = Date().addingTimeInterval(timeout)
|
||||
while Date() < deadline {
|
||||
if let failure = snapshotFailure() { throw failure }
|
||||
if snapshotStarted() { return }
|
||||
try await Task.sleep(nanoseconds: 20_000_000)
|
||||
}
|
||||
cancel()
|
||||
throw CloudASRError.transport("task-started timed out")
|
||||
}
|
||||
|
||||
func waitForFinalText(timeout: TimeInterval) async throws -> String {
|
||||
let deadline = Date().addingTimeInterval(timeout)
|
||||
while Date() < deadline {
|
||||
if let failure = snapshotFailure() { throw failure }
|
||||
if let text = snapshotFinalText() { return text }
|
||||
try await Task.sleep(nanoseconds: 20_000_000)
|
||||
}
|
||||
cancel()
|
||||
throw CloudASRError.transport("final result timed out")
|
||||
}
|
||||
|
||||
private func snapshotStarted() -> Bool {
|
||||
lock.withLock { started }
|
||||
}
|
||||
|
||||
private func snapshotFinalText() -> String? {
|
||||
lock.withLock { finalText }
|
||||
}
|
||||
|
||||
private func snapshotFailure() -> Error? {
|
||||
lock.withLock { failure }
|
||||
}
|
||||
|
||||
private func readLoop() async {
|
||||
var reducer = BailianASREventReducer()
|
||||
|
||||
while !Task.isCancelled {
|
||||
let message: URLSessionWebSocketTask.Message
|
||||
do {
|
||||
message = try await task.receive()
|
||||
} catch {
|
||||
publishFailure(CloudASRError.transport(error.localizedDescription))
|
||||
return
|
||||
}
|
||||
|
||||
let text: String
|
||||
switch message {
|
||||
case .string(let value):
|
||||
text = value
|
||||
case .data(let data):
|
||||
text = String(data: data, encoding: .utf8) ?? ""
|
||||
@unknown default:
|
||||
continue
|
||||
}
|
||||
guard !text.isEmpty else { continue }
|
||||
|
||||
switch reducer.apply(jsonText: text) {
|
||||
case .none:
|
||||
continue
|
||||
case .started:
|
||||
publishStarted()
|
||||
case .partial(let display):
|
||||
onPartial?(display)
|
||||
case .finished(let final):
|
||||
publishFinal(final)
|
||||
return
|
||||
case .failed(let message):
|
||||
publishFailure(CloudASRError.transport(message))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func publishStarted() {
|
||||
lock.withLock { started = true }
|
||||
}
|
||||
|
||||
private func publishFinal(_ text: String) {
|
||||
lock.withLock { finalText = text }
|
||||
}
|
||||
|
||||
private func publishFailure(_ error: Error) {
|
||||
lock.withLock { failure = error }
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,526 @@
|
||||
// CloudASRClients.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Provider-specific cloud ASR backends with personal-dictionary bias.
|
||||
|
||||
import Foundation
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public protocol CloudASRTranscribing: Sendable {
|
||||
func prepare(dictionary: PersonalDictionary) async throws
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String
|
||||
|
||||
/// Settings "validate connection" probe. Verifies transport + auth only.
|
||||
func probeConnection() async throws
|
||||
}
|
||||
|
||||
extension CloudASRTranscribing {
|
||||
/// Default probe: transcribe ~1 s of near-silence. An empty transcript
|
||||
/// counts as success — HTTP/streaming providers only need to prove that
|
||||
/// transport + auth work. Providers that reject silent/short audio or
|
||||
/// close after an empty final (DashScope realtime, Volcengine SAUC)
|
||||
/// override with a handshake-only probe.
|
||||
public func probeConnection() async throws {
|
||||
do {
|
||||
_ = try await transcribe(
|
||||
samples: [Float](repeating: 0.01, count: 16_000),
|
||||
sampleRate: 16_000,
|
||||
locale: Locale(identifier: "zh-CN"),
|
||||
dictionary: .empty
|
||||
)
|
||||
} catch CloudASRError.emptyTranscript {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public enum CloudASRClientFactory {
|
||||
public static func make(store: any ConfigurationStore, session: URLSession = .shared) -> CloudASRTranscribing {
|
||||
let providerId = store.asrProviderId
|
||||
let strategy = CloudASRModelCatalog.strategy(for: providerId)
|
||||
let asrModel = store.asrModel.isEmpty
|
||||
? CloudASRModelCatalog.defaultModel(for: providerId)
|
||||
: store.asrModel
|
||||
switch strategy {
|
||||
case .zhipuHotwords:
|
||||
return ZhipuCloudASRClient(
|
||||
apiKey: store.asrApiKey,
|
||||
model: asrModel,
|
||||
session: session
|
||||
)
|
||||
case .bailianStreaming:
|
||||
return BailianRealtimeASRClient(
|
||||
apiKey: store.asrApiKey,
|
||||
endpoint: store.asrBaseURL,
|
||||
model: asrModel,
|
||||
vocabularyID: nil,
|
||||
session: session
|
||||
)
|
||||
case .prompt:
|
||||
return PromptCloudASRClient(
|
||||
providerId: providerId,
|
||||
baseURL: store.asrBaseURL,
|
||||
apiKey: store.asrApiKey,
|
||||
model: asrModel,
|
||||
session: session
|
||||
)
|
||||
case .openRouterJson:
|
||||
return PromptCloudASRClient(
|
||||
providerId: providerId,
|
||||
baseURL: store.asrBaseURL,
|
||||
apiKey: store.asrApiKey,
|
||||
model: asrModel,
|
||||
session: session,
|
||||
requestFormat: .openRouterJson
|
||||
)
|
||||
case .volcengineStreaming:
|
||||
return VolcengineCloudASRClient(
|
||||
apiKey: store.asrApiKey,
|
||||
endpoint: store.asrBaseURL,
|
||||
resourceID: asrModel,
|
||||
session: session
|
||||
)
|
||||
case .openaiRealtimeStreaming:
|
||||
return OpenAIRealtimeASRClient(
|
||||
apiKey: store.asrApiKey,
|
||||
endpoint: store.asrBaseURL,
|
||||
model: asrModel,
|
||||
batchBaseURL: LLMProvider.provider(id: "openai").defaultBaseURL,
|
||||
session: session
|
||||
)
|
||||
case .localFallback:
|
||||
return UnsupportedCloudASRClient(providerId: providerId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Zhipu (hotwords + prompt)
|
||||
|
||||
struct ZhipuCloudASRClient: CloudASRTranscribing {
|
||||
let apiKey: String
|
||||
let model: String
|
||||
let session: URLSession
|
||||
|
||||
private static let maxDurationSeconds: TimeInterval = 30
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
let duration = Double(samples.count) / Double(sampleRate)
|
||||
guard duration <= Self.maxDurationSeconds else { throw CloudASRError.audioTooLong }
|
||||
|
||||
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
|
||||
let urlString = "https://open.bigmodel.cn/api/paas/v4\(CloudASRModelCatalog.zhipuTranscriptionPath)"
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
|
||||
let boundary = "Boundary-\(UUID().uuidString)"
|
||||
var body = Data()
|
||||
func appendField(_ name: String, _ value: String) {
|
||||
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Disposition: form-data; name=\"\(name)\"\r\n\r\n".data(using: .utf8)!)
|
||||
body.append("\(value)\r\n".data(using: .utf8)!)
|
||||
}
|
||||
|
||||
appendField("model", model)
|
||||
appendField("stream", "false")
|
||||
|
||||
let hotwords = dictionary.asrHotwords()
|
||||
if !hotwords.isEmpty,
|
||||
let hotwordsJSON = try? JSONSerialization.data(withJSONObject: hotwords),
|
||||
let hotwordsString = String(data: hotwordsJSON, encoding: .utf8) {
|
||||
appendField("hotwords", hotwordsString)
|
||||
}
|
||||
|
||||
let prompt = dictionary.asrPromptBias()
|
||||
if !prompt.isEmpty {
|
||||
appendField("prompt", prompt)
|
||||
}
|
||||
|
||||
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Type: audio/wav\r\n\r\n".data(using: .utf8)!)
|
||||
body.append(wav)
|
||||
body.append("\r\n".data(using: .utf8)!)
|
||||
body.append("--\(boundary)--\r\n".data(using: .utf8)!)
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
||||
request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")
|
||||
request.httpBody = body
|
||||
request.timeoutInterval = 60
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
try Self.validateHTTP(response: response, data: data)
|
||||
guard let text = Self.parseZhipuText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty else {
|
||||
throw CloudASRError.emptyTranscript
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
private static func parseZhipuText(from data: Data) -> String? {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
||||
return nil
|
||||
}
|
||||
return json["text"] as? String
|
||||
}
|
||||
|
||||
fileprivate static func validateHTTP(response: URLResponse, data: Data) throws {
|
||||
guard let http = response as? HTTPURLResponse else {
|
||||
throw CloudASRError.transport("non-HTTP response")
|
||||
}
|
||||
guard (200..<300).contains(http.statusCode) else {
|
||||
let message = parseErrorMessage(from: data)
|
||||
throw CloudASRError.http(status: http.statusCode, message: message)
|
||||
}
|
||||
}
|
||||
|
||||
fileprivate static func parseErrorMessage(from data: Data) -> String? {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
||||
return nil
|
||||
}
|
||||
if let message = json["message"] as? String { return message }
|
||||
if let error = json["error"] as? [String: Any],
|
||||
let message = error["message"] as? String {
|
||||
return message
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Alibaba Fun-ASR Flash (HTTP sync, context text bias)
|
||||
|
||||
struct AlibabaFunASRClient: CloudASRTranscribing {
|
||||
let apiKey: String
|
||||
let model: String
|
||||
let session: URLSession
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
|
||||
let dataURI = PCMSampleWavEncoder.dataURI(samples: samples, sampleRate: sampleRate)
|
||||
let urlString = CloudASRModelCatalog.alibabaAPIBase + CloudASRModelCatalog.alibabaMultimodalPath
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
|
||||
var messages: [[String: Any]] = []
|
||||
let context = dictionary.alibabaContextText()
|
||||
if !context.isEmpty {
|
||||
messages.append([
|
||||
"role": "user",
|
||||
"content": [
|
||||
["type": "input_text", "text": context],
|
||||
],
|
||||
])
|
||||
}
|
||||
messages.append([
|
||||
"role": "user",
|
||||
"content": [
|
||||
[
|
||||
"type": "input_audio",
|
||||
"input_audio": ["data": dataURI],
|
||||
],
|
||||
],
|
||||
])
|
||||
|
||||
let parameters: [String: Any] = [
|
||||
"format": "wav",
|
||||
"sample_rate": "\(sampleRate)",
|
||||
]
|
||||
|
||||
let body: [String: Any] = [
|
||||
"model": model,
|
||||
"input": ["messages": messages],
|
||||
"parameters": parameters,
|
||||
]
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
||||
request.setValue("disable", forHTTPHeaderField: "X-DashScope-SSE")
|
||||
request.httpBody = try JSONSerialization.data(withJSONObject: body)
|
||||
request.timeoutInterval = 90
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
||||
if let text = Self.parseText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty {
|
||||
return text
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
private static func parseText(from data: Data) -> String? {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||
let output = json["output"] as? [String: Any] else {
|
||||
return nil
|
||||
}
|
||||
if let text = output["text"] as? String { return text }
|
||||
if let sentence = output["sentence"] as? [String: Any],
|
||||
let text = sentence["text"] as? String {
|
||||
return text
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Prompt-biased transcription (OpenAI / MiMo / Groq / custom)
|
||||
|
||||
enum PromptCloudASRRequestFormat: Sendable {
|
||||
case multipart
|
||||
/// OpenRouter expects JSON `{ model, input_audio: { data, format } }`.
|
||||
case openRouterJson
|
||||
}
|
||||
|
||||
struct PromptCloudASRClient: CloudASRTranscribing {
|
||||
let providerId: String
|
||||
let baseURL: String
|
||||
let apiKey: String
|
||||
let model: String
|
||||
let session: URLSession
|
||||
var requestFormat: PromptCloudASRRequestFormat = .multipart
|
||||
|
||||
/// Groq / OpenRouter batch uploads cap around 30 s per request.
|
||||
private static let whisperCompatibleMaxDurationSeconds: TimeInterval = 30
|
||||
|
||||
init(
|
||||
providerId: String,
|
||||
baseURL: String,
|
||||
apiKey: String,
|
||||
model: String,
|
||||
session: URLSession,
|
||||
requestFormat: PromptCloudASRRequestFormat = .multipart
|
||||
) {
|
||||
self.providerId = providerId
|
||||
self.baseURL = baseURL
|
||||
self.apiKey = apiKey
|
||||
self.model = model
|
||||
self.session = session
|
||||
self.requestFormat = requestFormat
|
||||
}
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
if providerId == "mimo" {
|
||||
return try await transcribeMiMo(
|
||||
samples: samples,
|
||||
sampleRate: sampleRate,
|
||||
dictionary: dictionary
|
||||
)
|
||||
}
|
||||
if requestFormat == .openRouterJson {
|
||||
return try await transcribeOpenRouterJSON(
|
||||
samples: samples,
|
||||
sampleRate: sampleRate,
|
||||
dictionary: dictionary
|
||||
)
|
||||
}
|
||||
return try await transcribeOpenAIStyle(
|
||||
samples: samples,
|
||||
sampleRate: sampleRate,
|
||||
dictionary: dictionary
|
||||
)
|
||||
}
|
||||
|
||||
private func enforceWhisperDuration(samples: [Float], sampleRate: Int) throws {
|
||||
let duration = Double(samples.count) / Double(sampleRate)
|
||||
guard duration <= Self.whisperCompatibleMaxDurationSeconds else {
|
||||
throw CloudASRError.audioTooLong
|
||||
}
|
||||
}
|
||||
|
||||
private func transcribeOpenAIStyle(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
if providerId == "groq" || providerId == "openai" || providerId == "custom" {
|
||||
try enforceWhisperDuration(samples: samples, sampleRate: sampleRate)
|
||||
}
|
||||
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
|
||||
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
|
||||
let urlString = "\(trimmedBase)/audio/transcriptions"
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
|
||||
let boundary = "Boundary-\(UUID().uuidString)"
|
||||
var body = Data()
|
||||
func appendField(_ name: String, _ value: String) {
|
||||
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Disposition: form-data; name=\"\(name)\"\r\n\r\n".data(using: .utf8)!)
|
||||
body.append("\(value)\r\n".data(using: .utf8)!)
|
||||
}
|
||||
|
||||
appendField("model", model)
|
||||
let prompt = dictionary.asrPromptBias(maxCharacters: 600)
|
||||
if !prompt.isEmpty {
|
||||
appendField("prompt", prompt)
|
||||
}
|
||||
|
||||
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Type: audio/wav\r\n\r\n".data(using: .utf8)!)
|
||||
body.append(wav)
|
||||
body.append("\r\n".data(using: .utf8)!)
|
||||
body.append("--\(boundary)--\r\n".data(using: .utf8)!)
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
||||
request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")
|
||||
request.httpBody = body
|
||||
request.timeoutInterval = 90
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
||||
guard let text = Self.parseOpenAIText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty else {
|
||||
throw CloudASRError.emptyTranscript
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
private func transcribeOpenRouterJSON(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
try enforceWhisperDuration(samples: samples, sampleRate: sampleRate)
|
||||
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
|
||||
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
|
||||
let urlString = "\(trimmedBase)/audio/transcriptions"
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
|
||||
var body: [String: Any] = [
|
||||
"model": model,
|
||||
"input_audio": [
|
||||
"data": wav.base64EncodedString(),
|
||||
"format": "wav",
|
||||
],
|
||||
]
|
||||
let prompt = dictionary.asrPromptBias(maxCharacters: 600)
|
||||
if !prompt.isEmpty {
|
||||
body["prompt"] = prompt
|
||||
}
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
||||
request.httpBody = try JSONSerialization.data(withJSONObject: body)
|
||||
request.timeoutInterval = 90
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
||||
guard let text = Self.parseOpenAIText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty else {
|
||||
throw CloudASRError.emptyTranscript
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
private func transcribeMiMo(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
let dataURI = PCMSampleWavEncoder.dataURI(samples: samples, sampleRate: sampleRate)
|
||||
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
|
||||
let urlString = "\(trimmedBase)/chat/completions"
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
|
||||
var userContent: [[String: Any]] = []
|
||||
let prompt = dictionary.asrPromptBias()
|
||||
if !prompt.isEmpty {
|
||||
userContent.append(["type": "text", "text": prompt])
|
||||
}
|
||||
userContent.append([
|
||||
"type": "input_audio",
|
||||
"input_audio": ["data": dataURI],
|
||||
])
|
||||
|
||||
let body: [String: Any] = [
|
||||
"model": model,
|
||||
"messages": [
|
||||
["role": "user", "content": userContent],
|
||||
],
|
||||
"asr_options": ["language": "auto"],
|
||||
]
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
request.setValue(apiKey, forHTTPHeaderField: "api-key")
|
||||
request.httpBody = try JSONSerialization.data(withJSONObject: body)
|
||||
request.timeoutInterval = 90
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
||||
guard let text = Self.parseChatCompletionText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty else {
|
||||
throw CloudASRError.emptyTranscript
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
private static func parseOpenAIText(from data: Data) -> String? {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
||||
return nil
|
||||
}
|
||||
return json["text"] as? String
|
||||
}
|
||||
|
||||
private static func parseChatCompletionText(from data: Data) -> String? {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||
let choices = json["choices"] as? [[String: Any]],
|
||||
let first = choices.first,
|
||||
let message = first["message"] as? [String: Any] else {
|
||||
return nil
|
||||
}
|
||||
return message["content"] as? String
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Unsupported hosted ASR (Moonshot)
|
||||
|
||||
struct UnsupportedCloudASRClient: CloudASRTranscribing {
|
||||
let providerId: String
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
throw CloudASRError.providerUnsupported
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
// CloudASRConnectionCheck.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Settings "validate connection" probe shared by iOS and macOS.
|
||||
|
||||
import Foundation
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public enum CloudASRConnectionCheck {
|
||||
/// Verifies the active cloud ASR client can connect + authenticate.
|
||||
///
|
||||
/// Each backend decides how to probe (see `CloudASRTranscribing`):
|
||||
/// HTTP/batch providers transcribe a short silence clip and treat an
|
||||
/// empty transcript as success; Bailian / Volcengine / OpenAI Realtime
|
||||
/// handshake (and auth) only — silence clips make streaming backends fail.
|
||||
public static func validate(store: any ConfigurationStore) async throws {
|
||||
let client = CloudASRClientFactory.make(store: store)
|
||||
try await client.probeConnection()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
// CloudASRService.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Cloud-engine ASR: uploads PCM to the user's configured provider with
|
||||
// personal-dictionary bias. Streaming-capable providers use one utterance
|
||||
// WebSocket; others stay on chunked batch. Moonshot falls back to on-device ASR.
|
||||
|
||||
import Foundation
|
||||
import os
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public final class CloudASRService: ASRService, @unchecked Sendable {
|
||||
private let store: any ConfigurationStore
|
||||
private let session: URLSession
|
||||
private let localFallback: ASRService
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
private var client: CloudASRTranscribing?
|
||||
private var usesLocalFallback = false
|
||||
private var boundProviderId: String?
|
||||
private var cancelled = false
|
||||
private var streamingPipeline: StreamingUtterancePipeline?
|
||||
|
||||
public init(
|
||||
store: any ConfigurationStore = AppGroupStore(),
|
||||
session: URLSession = .shared,
|
||||
localFallback: ASRService? = nil
|
||||
) {
|
||||
self.store = store
|
||||
self.session = session
|
||||
// `SpeechAnalyzerASR` is internal, so it can't appear in a public
|
||||
// default argument value — resolve the fallback in the body instead.
|
||||
self.localFallback = localFallback ?? SpeechAnalyzerASR()
|
||||
}
|
||||
|
||||
/// Whether Flow should prefer utterance-level true streaming for the bound provider.
|
||||
public var supportsUtteranceStreaming: Bool {
|
||||
CloudASRModelCatalog.supportsTrueStreamingASR(for: store.asrProviderId)
|
||||
}
|
||||
|
||||
public func resetForNewUtterance() {
|
||||
lock.withLock { cancelled = false }
|
||||
if usesLocalFallback {
|
||||
localFallback.resetForNewUtterance()
|
||||
}
|
||||
}
|
||||
|
||||
public func warmup(locale: Locale) async {
|
||||
bindClientIfNeeded()
|
||||
if usesLocalFallback {
|
||||
await localFallback.warmup(locale: locale)
|
||||
return
|
||||
}
|
||||
guard let client = lock.withLock({ client }) else { return }
|
||||
do {
|
||||
try await client.prepare(dictionary: store.personalDictionary)
|
||||
} catch {
|
||||
OSGLog.asr.warning("cloud ASR vocabulary prepare failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
|
||||
guard !samples.isEmpty else { return .success("") }
|
||||
if Task.isCancelled || lock.withLock({ cancelled }) { return .cancelled }
|
||||
|
||||
bindClientIfNeeded()
|
||||
if usesLocalFallback {
|
||||
return await localFallback.transcribeChunk(samples: samples, locale: locale)
|
||||
}
|
||||
|
||||
guard let client = lock.withLock({ client }) else {
|
||||
return .failure(CloudASRError.providerUnsupported.localizedDescription)
|
||||
}
|
||||
|
||||
let startedAt = Date()
|
||||
do {
|
||||
let text = try await client.transcribe(
|
||||
samples: samples,
|
||||
sampleRate: 16_000,
|
||||
locale: locale,
|
||||
dictionary: store.personalDictionary
|
||||
)
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
FlowTrace.transcript(
|
||||
"asr.cloud.chunk",
|
||||
trimmed,
|
||||
"engine=cloud provider=\(store.asrProviderId) samples=\(samples.count) "
|
||||
+ "rms=\(FlowTrace.rms(samples)) elapsed=\(FlowTrace.seconds(since: startedAt))s"
|
||||
)
|
||||
return trimmed.isEmpty ? .success("") : .success(trimmed)
|
||||
} catch is CancellationError {
|
||||
FlowTrace.asr("cloud.chunk.cancelled", "samples=\(samples.count)")
|
||||
return .cancelled
|
||||
} catch {
|
||||
FlowTrace.warn(
|
||||
"asr.cloud.chunk.failed",
|
||||
"provider=\(store.asrProviderId) samples=\(samples.count) "
|
||||
+ "rms=\(FlowTrace.rms(samples)) elapsed=\(FlowTrace.seconds(since: startedAt))s "
|
||||
+ "error=\(error.localizedDescription)"
|
||||
)
|
||||
return .failure(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
/// Utterance-level streaming; if the session cannot start, fall back to
|
||||
/// chunked batch on the same mic stream. Mid-stream failures surface as
|
||||
/// errors (finalize still has PCM batch fallback).
|
||||
public func transcribeUtteranceStreaming(
|
||||
stream: AsyncStream<AudioBufferSnapshot>,
|
||||
locale: Locale,
|
||||
onPartial: @escaping @Sendable (String) -> Void
|
||||
) async -> ChunkedUtterancePipelineOutcome {
|
||||
bindClientIfNeeded()
|
||||
if usesLocalFallback {
|
||||
let pipeline = ChunkedUtterancePipeline(asr: localFallback, locale: locale)
|
||||
return await pipeline.transcribe(stream: stream, onPartial: onPartial)
|
||||
}
|
||||
|
||||
guard let streamingClient = lock.withLock({ client as? CloudASRStreamingCapable }) else {
|
||||
let pipeline = ChunkedUtterancePipeline(asr: self, locale: locale)
|
||||
return await pipeline.transcribe(stream: stream, onPartial: onPartial)
|
||||
}
|
||||
|
||||
let session: any CloudASRStreamingSession
|
||||
do {
|
||||
session = try await streamingClient.openStreamingSession(
|
||||
locale: locale,
|
||||
dictionary: store.personalDictionary,
|
||||
onPartial: onPartial
|
||||
)
|
||||
} catch {
|
||||
OSGLog.asr.warning(
|
||||
"streaming ASR session open failed, using chunked batch: \(error.localizedDescription, privacy: .public)"
|
||||
)
|
||||
let pipeline = ChunkedUtterancePipeline(asr: self, locale: locale)
|
||||
return await pipeline.transcribe(stream: stream, onPartial: onPartial)
|
||||
}
|
||||
|
||||
let pipeline = StreamingUtterancePipeline(
|
||||
client: streamingClient,
|
||||
locale: locale,
|
||||
dictionary: store.personalDictionary
|
||||
)
|
||||
lock.withLock { streamingPipeline = pipeline }
|
||||
let outcome = await pipeline.transcribe(
|
||||
stream: stream,
|
||||
onPartial: onPartial,
|
||||
preopenedSession: session
|
||||
)
|
||||
lock.withLock { streamingPipeline = nil }
|
||||
return outcome
|
||||
}
|
||||
|
||||
public func transcribe(
|
||||
stream: AsyncStream<AudioBufferSnapshot>,
|
||||
locale: Locale
|
||||
) -> AsyncStream<ASREvent> {
|
||||
bindClientIfNeeded()
|
||||
if usesLocalFallback {
|
||||
return localFallback.transcribe(stream: stream, locale: locale)
|
||||
}
|
||||
|
||||
if supportsUtteranceStreaming, lock.withLock({ client is CloudASRStreamingCapable }) {
|
||||
return AsyncStream { continuation in
|
||||
continuation.yield(.capability(onDeviceSupported: false))
|
||||
let task = Task {
|
||||
let outcome = await self.transcribeUtteranceStreaming(
|
||||
stream: stream,
|
||||
locale: locale,
|
||||
onPartial: { partial in
|
||||
continuation.yield(.partial(partial))
|
||||
}
|
||||
)
|
||||
switch outcome {
|
||||
case .success(let success):
|
||||
continuation.yield(.final(success.text))
|
||||
case .failure(let message):
|
||||
continuation.yield(.error(message))
|
||||
case .cancelled:
|
||||
break
|
||||
}
|
||||
continuation.finish()
|
||||
}
|
||||
continuation.onTermination = { @Sendable _ in
|
||||
task.cancel()
|
||||
self.cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return AsyncStream { continuation in
|
||||
continuation.yield(.capability(onDeviceSupported: false))
|
||||
let task = Task {
|
||||
var samples: [Float] = []
|
||||
for await snap in stream {
|
||||
if Task.isCancelled { break }
|
||||
samples.append(contentsOf: snap.samples)
|
||||
}
|
||||
guard !Task.isCancelled, !self.lock.withLock({ self.cancelled }) else {
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
guard !samples.isEmpty else {
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
|
||||
switch await self.transcribeChunk(samples: samples, locale: locale) {
|
||||
case .success(let text):
|
||||
if text.isEmpty {
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
|
||||
} else {
|
||||
continuation.yield(.final(text))
|
||||
}
|
||||
case .failure(let message):
|
||||
continuation.yield(.error(message))
|
||||
case .cancelled:
|
||||
break
|
||||
}
|
||||
continuation.finish()
|
||||
}
|
||||
continuation.onTermination = { @Sendable _ in
|
||||
task.cancel()
|
||||
self.cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public func cancel() {
|
||||
lock.withLock { cancelled = true }
|
||||
let pipeline = lock.withLock { streamingPipeline }
|
||||
Task { await pipeline?.cancel() }
|
||||
localFallback.cancel()
|
||||
}
|
||||
|
||||
private func bindClientIfNeeded() {
|
||||
let providerId = store.asrProviderId
|
||||
let strategy = CloudASRModelCatalog.strategy(for: providerId)
|
||||
lock.withLock {
|
||||
guard boundProviderId != providerId else { return }
|
||||
boundProviderId = providerId
|
||||
usesLocalFallback = strategy == .localFallback
|
||||
client = usesLocalFallback
|
||||
? nil
|
||||
: CloudASRClientFactory.make(store: store, session: session)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
// CloudASRStreaming.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Utterance-scoped cloud ASR sessions: one long-lived connection per press,
|
||||
// streaming PCM up and interim text down. Chunked batch ASR remains the
|
||||
// fallback for providers without a true streaming protocol.
|
||||
|
||||
import Foundation
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
/// Long-lived cloud ASR session for one Flow utterance.
|
||||
public protocol CloudASRStreamingSession: Sendable {
|
||||
/// Append 16 kHz mono Float32 PCM captured while the mic is open.
|
||||
func append(samples: [Float]) async throws
|
||||
/// Signal end-of-audio and wait for the polish-ready final transcript.
|
||||
func finish() async throws -> String
|
||||
func cancel()
|
||||
}
|
||||
|
||||
/// Providers that can open an utterance-level streaming session.
|
||||
public protocol CloudASRStreamingCapable: CloudASRTranscribing {
|
||||
func openStreamingSession(
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary,
|
||||
onPartial: @escaping @Sendable (String) -> Void
|
||||
) async throws -> any CloudASRStreamingSession
|
||||
}
|
||||
|
||||
/// Feeds a live mic stream into a cloud streaming session and mirrors the
|
||||
/// existing `ChunkedUtterancePipelineOutcome` surface for Flow.
|
||||
public actor StreamingUtterancePipeline {
|
||||
private let client: any CloudASRStreamingCapable
|
||||
private let locale: Locale
|
||||
private let dictionary: PersonalDictionary
|
||||
private var cancelled = false
|
||||
private var activeSession: (any CloudASRStreamingSession)?
|
||||
|
||||
public init(
|
||||
client: any CloudASRStreamingCapable,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) {
|
||||
self.client = client
|
||||
self.locale = locale
|
||||
self.dictionary = dictionary
|
||||
}
|
||||
|
||||
public func cancel() {
|
||||
cancelled = true
|
||||
activeSession?.cancel()
|
||||
}
|
||||
|
||||
public func transcribe(
|
||||
stream: AsyncStream<AudioBufferSnapshot>,
|
||||
onPartial: @Sendable @escaping (String) -> Void,
|
||||
preopenedSession: (any CloudASRStreamingSession)? = nil
|
||||
) async -> ChunkedUtterancePipelineOutcome {
|
||||
cancelled = false
|
||||
let startedAt = Date()
|
||||
// Counted so an empty cloud transcript can be told apart from "we never
|
||||
// uploaded any audio" — the two look identical to the user.
|
||||
var uploadedSamples = 0
|
||||
var uploadedSnapshots = 0
|
||||
do {
|
||||
let session: any CloudASRStreamingSession
|
||||
if let preopenedSession {
|
||||
session = preopenedSession
|
||||
} else {
|
||||
session = try await client.openStreamingSession(
|
||||
locale: locale,
|
||||
dictionary: dictionary,
|
||||
onPartial: onPartial
|
||||
)
|
||||
}
|
||||
activeSession = session
|
||||
FlowTrace.asr(
|
||||
"cloud.stream.opened",
|
||||
"locale=\(locale.identifier(.bcp47)) preopened=\(preopenedSession != nil ? 1 : 0)"
|
||||
)
|
||||
|
||||
for await snap in stream {
|
||||
if cancelled || Task.isCancelled {
|
||||
session.cancel()
|
||||
FlowTrace.asr(
|
||||
"cloud.stream.cancelledMidUpload",
|
||||
"uploadedSamples=\(uploadedSamples)"
|
||||
)
|
||||
return .cancelled
|
||||
}
|
||||
guard !snap.samples.isEmpty else { continue }
|
||||
uploadedSnapshots += 1
|
||||
uploadedSamples += snap.samples.count
|
||||
try await session.append(samples: snap.samples)
|
||||
}
|
||||
|
||||
FlowTrace.asr(
|
||||
"cloud.stream.uploadDone",
|
||||
"snapshots=\(uploadedSnapshots) samples=\(uploadedSamples) "
|
||||
+ "seconds=\(FlowTrace.seconds(samples: uploadedSamples))"
|
||||
)
|
||||
|
||||
if cancelled || Task.isCancelled {
|
||||
session.cancel()
|
||||
return .cancelled
|
||||
}
|
||||
|
||||
let finalText = try await session.finish()
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
activeSession = nil
|
||||
guard !finalText.isEmpty else {
|
||||
FlowTrace.warn(
|
||||
"asr.cloud.stream.emptyFinal",
|
||||
"uploadedSamples=\(uploadedSamples) "
|
||||
+ "seconds=\(FlowTrace.seconds(samples: uploadedSamples)) "
|
||||
+ "elapsed=\(FlowTrace.seconds(since: startedAt))s"
|
||||
)
|
||||
return .failure(SharedL10n.string("error.asr.noSpeech"))
|
||||
}
|
||||
FlowTrace.transcript(
|
||||
"asr.cloud.final",
|
||||
finalText,
|
||||
"engine=cloud uploadedSamples=\(uploadedSamples) "
|
||||
+ "elapsed=\(FlowTrace.seconds(since: startedAt))s"
|
||||
)
|
||||
return .success(ChunkedUtteranceSuccess(text: finalText))
|
||||
} catch is CancellationError {
|
||||
activeSession?.cancel()
|
||||
activeSession = nil
|
||||
FlowTrace.asr("cloud.stream.cancelled", "uploadedSamples=\(uploadedSamples)")
|
||||
return .cancelled
|
||||
} catch {
|
||||
activeSession?.cancel()
|
||||
activeSession = nil
|
||||
if cancelled || Task.isCancelled { return .cancelled }
|
||||
FlowTrace.warn(
|
||||
"asr.cloud.stream.failed",
|
||||
"uploadedSamples=\(uploadedSamples) "
|
||||
+ "elapsed=\(FlowTrace.seconds(since: startedAt))s "
|
||||
+ "error=\(error.localizedDescription)"
|
||||
)
|
||||
return .failure(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared PCM helpers for streaming cloud clients.
|
||||
enum CloudASRStreamingPCM {
|
||||
static func pcm16LE(samples: [Float]) -> Data {
|
||||
var data = Data()
|
||||
data.reserveCapacity(samples.count * 2)
|
||||
for sample in samples {
|
||||
let scaled = sample * 32_767.0
|
||||
let clipped = Swift.max(-32_768.0, Swift.min(32_767.0, scaled))
|
||||
var littleEndian = Int16(clipped.rounded()).littleEndian
|
||||
withUnsafeBytes(of: &littleEndian) { data.append(contentsOf: $0) }
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
/// Linear upsample 16 kHz → 24 kHz for OpenAI Realtime PCM input.
|
||||
static func upsample16kTo24k(_ samples: [Float]) -> [Float] {
|
||||
guard !samples.isEmpty else { return [] }
|
||||
let outCount = max(1, samples.count * 3 / 2)
|
||||
var output = [Float]()
|
||||
output.reserveCapacity(outCount)
|
||||
let lastIndex = samples.count - 1
|
||||
for i in 0..<outCount {
|
||||
let src = Double(i) * 16.0 / 24.0
|
||||
let i0 = min(Int(src), lastIndex)
|
||||
let i1 = min(i0 + 1, lastIndex)
|
||||
let frac = Float(src - Double(i0))
|
||||
output.append(samples[i0] + (samples[i1] - samples[i0]) * frac)
|
||||
}
|
||||
return output
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
// CloudASRStreamingEventParsing.swift
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Pure reducers for streaming ASR WebSocket event JSON — hermetic fixtures
|
||||
// without a live socket.
|
||||
|
||||
import Foundation
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
// MARK: - Bailian Fun-ASR Realtime
|
||||
|
||||
enum BailianASREventEffect: Equatable, Sendable {
|
||||
case none
|
||||
case started
|
||||
case partial(String)
|
||||
case finished(String)
|
||||
case failed(String)
|
||||
}
|
||||
|
||||
struct BailianASREventReducer: Sendable {
|
||||
private(set) var finalSegments: [Int64: String] = [:]
|
||||
private(set) var partialSegments: [Int64: String] = [:]
|
||||
private(set) var lastResultText = ""
|
||||
private(set) var started = false
|
||||
|
||||
mutating func apply(jsonText: String) -> BailianASREventEffect {
|
||||
guard let data = jsonText.data(using: .utf8),
|
||||
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
||||
return .none
|
||||
}
|
||||
return apply(json: json)
|
||||
}
|
||||
|
||||
mutating func apply(json: [String: Any]) -> BailianASREventEffect {
|
||||
guard let header = json["header"] as? [String: Any] else { return .none }
|
||||
let event = header["event"] as? String ?? ""
|
||||
|
||||
switch event {
|
||||
case "task-started":
|
||||
started = true
|
||||
return .started
|
||||
case "result-generated":
|
||||
return applyResultGenerated(json: json)
|
||||
case "task-finished":
|
||||
if finalSegments.isEmpty {
|
||||
return .finished(lastResultText)
|
||||
}
|
||||
let ordered = finalSegments.keys.sorted().compactMap { finalSegments[$0] }
|
||||
return .finished(BailianRealtimeASRClient.mergeSegments(ordered))
|
||||
case "task-failed":
|
||||
let message = header["error_message"] as? String ?? "task failed"
|
||||
return .failed(message)
|
||||
default:
|
||||
return .none
|
||||
}
|
||||
}
|
||||
|
||||
private mutating func applyResultGenerated(json: [String: Any]) -> BailianASREventEffect {
|
||||
guard let payload = json["payload"] as? [String: Any],
|
||||
let output = payload["output"] as? [String: Any],
|
||||
let sentenceObj = output["sentence"] as? [String: Any] else {
|
||||
return .none
|
||||
}
|
||||
if sentenceObj["heartbeat"] as? Bool == true { return .none }
|
||||
guard let rawText = sentenceObj["text"] as? String else { return .none }
|
||||
let trimmed = rawText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return .none }
|
||||
|
||||
lastResultText = trimmed
|
||||
let sentenceID = sentenceObj["sentence_id"] as? Int64 ?? 0
|
||||
let sentenceEndValue = sentenceObj["sentence_end"]
|
||||
let sentenceEnd = sentenceEndValue as? Bool ?? false
|
||||
let endTime = sentenceObj["end_time"] as? Int64 ?? 0
|
||||
let isFinal = sentenceEndValue != nil ? sentenceEnd : endTime > 0
|
||||
|
||||
if isFinal {
|
||||
finalSegments[sentenceID] = trimmed
|
||||
partialSegments.removeValue(forKey: sentenceID)
|
||||
} else {
|
||||
partialSegments[sentenceID] = trimmed
|
||||
}
|
||||
|
||||
var displayParts: [String] = []
|
||||
let ids = Set(finalSegments.keys).union(partialSegments.keys).sorted()
|
||||
for id in ids {
|
||||
if let committed = finalSegments[id] {
|
||||
displayParts.append(committed)
|
||||
} else if let live = partialSegments[id] {
|
||||
displayParts.append(live)
|
||||
}
|
||||
}
|
||||
let display = BailianRealtimeASRClient.mergeSegments(displayParts)
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return display.isEmpty ? .none : .partial(display)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - OpenAI Realtime transcription
|
||||
|
||||
enum OpenAIRealtimeEventEffect: Equatable, Sendable {
|
||||
case none
|
||||
case sessionReady
|
||||
case partial(String)
|
||||
case failed(String)
|
||||
}
|
||||
|
||||
struct OpenAIRealtimeTranscriptReducer: Sendable {
|
||||
private(set) var sessionReady = false
|
||||
private(set) var partialByItem: [String: String] = [:]
|
||||
private(set) var completedByItem: [String: String] = [:]
|
||||
private(set) var itemOrder: [String] = []
|
||||
|
||||
mutating func apply(jsonText: String) -> OpenAIRealtimeEventEffect {
|
||||
guard let data = jsonText.data(using: .utf8),
|
||||
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||
let type = json["type"] as? String else {
|
||||
return .none
|
||||
}
|
||||
return apply(type: type, json: json)
|
||||
}
|
||||
|
||||
mutating func apply(type: String, json: [String: Any]) -> OpenAIRealtimeEventEffect {
|
||||
switch type {
|
||||
case "session.created", "session.updated":
|
||||
sessionReady = true
|
||||
return .sessionReady
|
||||
case "conversation.item.input_audio_transcription.delta":
|
||||
let itemID = json["item_id"] as? String ?? "default"
|
||||
let delta = json["delta"] as? String ?? ""
|
||||
guard !delta.isEmpty else { return .none }
|
||||
if partialByItem[itemID] == nil, completedByItem[itemID] == nil {
|
||||
itemOrder.append(itemID)
|
||||
}
|
||||
partialByItem[itemID, default: ""] += delta
|
||||
let display = composedDisplay()
|
||||
return display.isEmpty ? .none : .partial(display)
|
||||
case "conversation.item.input_audio_transcription.completed":
|
||||
let itemID = json["item_id"] as? String ?? "default"
|
||||
let transcript = (json["transcript"] as? String ?? "")
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !itemOrder.contains(itemID) {
|
||||
itemOrder.append(itemID)
|
||||
}
|
||||
if !transcript.isEmpty {
|
||||
completedByItem[itemID] = transcript
|
||||
}
|
||||
partialByItem.removeValue(forKey: itemID)
|
||||
let display = composedDisplay()
|
||||
return display.isEmpty ? .none : .partial(display)
|
||||
case "error":
|
||||
let message: String
|
||||
if let error = json["error"] as? [String: Any],
|
||||
let nested = error["message"] as? String {
|
||||
message = nested
|
||||
} else {
|
||||
message = json["message"] as? String ?? "OpenAI realtime error"
|
||||
}
|
||||
return .failed(message)
|
||||
default:
|
||||
return .none
|
||||
}
|
||||
}
|
||||
|
||||
func composedDisplay() -> String {
|
||||
itemOrder.compactMap { id in
|
||||
completedByItem[id] ?? partialByItem[id]
|
||||
}
|
||||
.joined(separator: " ")
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
|
||||
func composedFinal() -> String {
|
||||
itemOrder.compactMap { completedByItem[$0] }
|
||||
.joined(separator: " ")
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
|
||||
static func languageHint(from locale: Locale) -> String? {
|
||||
let id = locale.identifier.lowercased()
|
||||
if id.hasPrefix("zh") { return "zh" }
|
||||
if id.hasPrefix("en") { return "en" }
|
||||
if id.hasPrefix("ja") { return "ja" }
|
||||
if id.hasPrefix("ko") { return "ko" }
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
// OpenAIRealtimeASRClient.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// OpenAI Realtime transcription (WebSocket). Streams PCM and transcript
|
||||
// deltas for utterance-level ASR. Batch `/audio/transcriptions` remains the
|
||||
// fallback path when realtime is unavailable.
|
||||
|
||||
import Foundation
|
||||
import os
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
struct OpenAIRealtimeASRClient: CloudASRTranscribing, CloudASRStreamingCapable {
|
||||
let apiKey: String
|
||||
let endpoint: String
|
||||
let model: String
|
||||
let session: URLSession
|
||||
/// Used when streaming fails and Flow falls back to chunked batch ASR.
|
||||
private let batchClient: PromptCloudASRClient
|
||||
|
||||
static let appendChunkBytes = 4_800 // 100 ms @ 24 kHz / 16-bit mono.
|
||||
static let finalTimeout: TimeInterval = 15
|
||||
|
||||
init(
|
||||
apiKey: String,
|
||||
endpoint: String,
|
||||
model: String,
|
||||
batchBaseURL: String,
|
||||
session: URLSession
|
||||
) {
|
||||
self.apiKey = apiKey
|
||||
self.endpoint = endpoint
|
||||
self.model = model
|
||||
self.session = session
|
||||
self.batchClient = PromptCloudASRClient(
|
||||
providerId: "openai",
|
||||
baseURL: batchBaseURL.isEmpty ? "https://api.openai.com/v1" : batchBaseURL,
|
||||
apiKey: apiKey,
|
||||
model: Self.batchModel(from: model),
|
||||
session: session
|
||||
)
|
||||
}
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func openStreamingSession(
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary,
|
||||
onPartial: @escaping @Sendable (String) -> Void
|
||||
) async throws -> any CloudASRStreamingSession {
|
||||
_ = dictionary
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
let url = try resolvedEndpointURL()
|
||||
var request = URLRequest(url: url)
|
||||
request.timeoutInterval = 8
|
||||
request.setValue(
|
||||
"Bearer \(apiKey.trimmingCharacters(in: .whitespacesAndNewlines))",
|
||||
forHTTPHeaderField: "Authorization"
|
||||
)
|
||||
|
||||
let wsTask = session.webSocketTask(with: request)
|
||||
wsTask.resume()
|
||||
let live = OpenAIRealtimeStreamingSession(
|
||||
wsTask: wsTask,
|
||||
model: resolvedRealtimeModel,
|
||||
locale: locale,
|
||||
onPartial: onPartial
|
||||
)
|
||||
try await live.start()
|
||||
return live
|
||||
}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
try await batchClient.transcribe(
|
||||
samples: samples,
|
||||
sampleRate: sampleRate,
|
||||
locale: locale,
|
||||
dictionary: dictionary
|
||||
)
|
||||
}
|
||||
|
||||
func probeConnection() async throws {
|
||||
do {
|
||||
let session = try await openStreamingSession(
|
||||
locale: Locale(identifier: "zh-CN"),
|
||||
dictionary: .empty,
|
||||
onPartial: { _ in }
|
||||
)
|
||||
session.cancel()
|
||||
} catch {
|
||||
try await batchClient.probeConnection()
|
||||
}
|
||||
}
|
||||
|
||||
private var resolvedRealtimeModel: String {
|
||||
let trimmed = model.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if trimmed.isEmpty || trimmed.hasPrefix("gpt-4o") || trimmed == "whisper-1" {
|
||||
return CloudASRModelCatalog.openAIRealtimeWhisper
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
private func resolvedEndpointURL() throws -> URL {
|
||||
let raw = endpoint.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if raw.hasPrefix("wss://") || raw.hasPrefix("ws://") {
|
||||
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
|
||||
return url
|
||||
}
|
||||
guard let url = URL(string: CloudASRModelCatalog.openAIRealtimeEndpoint) else {
|
||||
throw CloudASRError.invalidURL
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
private static func batchModel(from model: String) -> String {
|
||||
let trimmed = model.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if trimmed.isEmpty || trimmed.contains("realtime") {
|
||||
return CloudASRModelCatalog.openAITranscribe
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Utterance session
|
||||
|
||||
private final class OpenAIRealtimeStreamingSession: CloudASRStreamingSession, @unchecked Sendable {
|
||||
private let wsTask: URLSessionWebSocketTask
|
||||
private let model: String
|
||||
private let locale: Locale
|
||||
private let onPartial: @Sendable (String) -> Void
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
private var receiveTask: Task<Void, Never>?
|
||||
private var failure: Error?
|
||||
private var finished = false
|
||||
private var pcmBuffer = Data()
|
||||
private var reducer = OpenAIRealtimeTranscriptReducer()
|
||||
private var awaitingCommit = false
|
||||
|
||||
init(
|
||||
wsTask: URLSessionWebSocketTask,
|
||||
model: String,
|
||||
locale: Locale,
|
||||
onPartial: @escaping @Sendable (String) -> Void
|
||||
) {
|
||||
self.wsTask = wsTask
|
||||
self.model = model
|
||||
self.locale = locale
|
||||
self.onPartial = onPartial
|
||||
}
|
||||
|
||||
func start() async throws {
|
||||
receiveTask = Task { [weak self] in
|
||||
await self?.receiveLoop()
|
||||
}
|
||||
let language = OpenAIRealtimeTranscriptReducer.languageHint(from: locale)
|
||||
var transcription: [String: Any] = [
|
||||
"model": model,
|
||||
"delay": "low",
|
||||
]
|
||||
if let language {
|
||||
transcription["language"] = language
|
||||
}
|
||||
var input: [String: Any] = [
|
||||
"format": [
|
||||
"type": "audio/pcm",
|
||||
"rate": 24_000,
|
||||
],
|
||||
"transcription": transcription,
|
||||
]
|
||||
input["turn_detection"] = NSNull()
|
||||
let update: [String: Any] = [
|
||||
"type": "session.update",
|
||||
"session": [
|
||||
"type": "transcription",
|
||||
"audio": [
|
||||
"input": input,
|
||||
],
|
||||
],
|
||||
]
|
||||
try await sendJSON(update)
|
||||
let deadline = Date().addingTimeInterval(8)
|
||||
while Date() < deadline {
|
||||
try throwIfFailed()
|
||||
if lock.withLock({ reducer.sessionReady }) { return }
|
||||
try await Task.sleep(nanoseconds: 20_000_000)
|
||||
}
|
||||
cancel()
|
||||
throw CloudASRError.transport("OpenAI realtime session timed out")
|
||||
}
|
||||
|
||||
func append(samples: [Float]) async throws {
|
||||
try throwIfFailed()
|
||||
let upsampled = CloudASRStreamingPCM.upsample16kTo24k(samples)
|
||||
let pcm = CloudASRStreamingPCM.pcm16LE(samples: upsampled)
|
||||
let frames: [Data] = lock.withLock {
|
||||
pcmBuffer.append(pcm)
|
||||
var frames: [Data] = []
|
||||
while pcmBuffer.count >= OpenAIRealtimeASRClient.appendChunkBytes {
|
||||
let frame = pcmBuffer.prefix(OpenAIRealtimeASRClient.appendChunkBytes)
|
||||
frames.append(Data(frame))
|
||||
pcmBuffer.removeFirst(OpenAIRealtimeASRClient.appendChunkBytes)
|
||||
}
|
||||
return frames
|
||||
}
|
||||
for frame in frames {
|
||||
try await sendAppend(frame)
|
||||
}
|
||||
}
|
||||
|
||||
func finish() async throws -> String {
|
||||
try throwIfFailed()
|
||||
let trailing: Data = lock.withLock {
|
||||
let data = pcmBuffer
|
||||
pcmBuffer.removeAll(keepingCapacity: false)
|
||||
awaitingCommit = true
|
||||
return data
|
||||
}
|
||||
if !trailing.isEmpty {
|
||||
try await sendAppend(trailing)
|
||||
}
|
||||
try await sendJSON(["type": "input_audio_buffer.commit"])
|
||||
|
||||
let deadline = Date().addingTimeInterval(OpenAIRealtimeASRClient.finalTimeout)
|
||||
while Date() < deadline {
|
||||
try throwIfFailed()
|
||||
let snapshot = lock.withLock {
|
||||
(awaitingCommit, reducer.composedFinal(), reducer.composedDisplay())
|
||||
}
|
||||
if !snapshot.0 {
|
||||
let text = snapshot.1.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? snapshot.2
|
||||
: snapshot.1
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
cancel()
|
||||
if trimmed.isEmpty { throw CloudASRError.emptyTranscript }
|
||||
return trimmed
|
||||
}
|
||||
let settled = lock.withLock {
|
||||
!reducer.completedByItem.isEmpty && reducer.partialByItem.isEmpty && !awaitingCommit
|
||||
}
|
||||
if settled {
|
||||
let text = lock.withLock { reducer.composedFinal() }
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
cancel()
|
||||
if text.isEmpty { throw CloudASRError.emptyTranscript }
|
||||
return text
|
||||
}
|
||||
try await Task.sleep(nanoseconds: 20_000_000)
|
||||
}
|
||||
let fallback = lock.withLock {
|
||||
let final = reducer.composedFinal()
|
||||
return final.isEmpty ? reducer.composedDisplay() : final
|
||||
}
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
cancel()
|
||||
if fallback.isEmpty {
|
||||
throw CloudASRError.transport("OpenAI realtime final timed out")
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
receiveTask?.cancel()
|
||||
wsTask.cancel(with: .normalClosure, reason: nil)
|
||||
lock.withLock { finished = true }
|
||||
}
|
||||
|
||||
private func receiveLoop() async {
|
||||
while !Task.isCancelled {
|
||||
let message: URLSessionWebSocketTask.Message
|
||||
do {
|
||||
message = try await wsTask.receive()
|
||||
} catch {
|
||||
publishFailure(CloudASRError.transport(error.localizedDescription))
|
||||
return
|
||||
}
|
||||
let text: String
|
||||
switch message {
|
||||
case .string(let value):
|
||||
text = value
|
||||
case .data(let data):
|
||||
text = String(data: data, encoding: .utf8) ?? ""
|
||||
@unknown default:
|
||||
continue
|
||||
}
|
||||
guard !text.isEmpty else { continue }
|
||||
|
||||
let effect = lock.withLock { () -> OpenAIRealtimeEventEffect in
|
||||
let effect = reducer.apply(jsonText: text)
|
||||
if case .partial = effect, !reducer.completedByItem.isEmpty {
|
||||
awaitingCommit = false
|
||||
}
|
||||
return effect
|
||||
}
|
||||
switch effect {
|
||||
case .none, .sessionReady:
|
||||
continue
|
||||
case .partial(let display):
|
||||
onPartial(display)
|
||||
case .failed(let message):
|
||||
publishFailure(CloudASRError.transport(message))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func sendAppend(_ pcm: Data) async throws {
|
||||
let audio = pcm.base64EncodedString()
|
||||
try await sendJSON([
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": audio,
|
||||
])
|
||||
}
|
||||
|
||||
private func sendJSON(_ body: [String: Any]) async throws {
|
||||
guard JSONSerialization.isValidJSONObject(body),
|
||||
let data = try? JSONSerialization.data(withJSONObject: body),
|
||||
let string = String(data: data, encoding: .utf8) else {
|
||||
throw CloudASRError.decoding("invalid realtime payload")
|
||||
}
|
||||
do {
|
||||
try await wsTask.send(.string(string))
|
||||
} catch {
|
||||
throw CloudASRError.transport(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private func throwIfFailed() throws {
|
||||
let (error, done) = lock.withLock { (failure, finished) }
|
||||
if let error { throw error }
|
||||
if done { throw CloudASRError.transport("OpenAI realtime session cancelled") }
|
||||
}
|
||||
|
||||
private func publishFailure(_ error: Error) {
|
||||
lock.withLock { failure = error }
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,578 @@
|
||||
// VolcengineCloudASRClient.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Volcengine SAUC bigmodel ASR client. Utterance-level WebSocket session with
|
||||
// enable_nonstream (official two-pass): interim text for on-screen partials,
|
||||
// definite utterances for polish-ready finals.
|
||||
|
||||
import Foundation
|
||||
import os
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
struct VolcengineCloudASRClient: CloudASRTranscribing, CloudASRStreamingCapable {
|
||||
let apiKey: String
|
||||
let endpoint: String
|
||||
let resourceID: String
|
||||
let session: URLSession
|
||||
|
||||
static let targetChunkBytes = 6_400 // 200 ms @ 16 kHz, 16-bit, mono.
|
||||
static let finalTimeout: TimeInterval = 12
|
||||
private static let hotwordCap = 80
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func openStreamingSession(
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary,
|
||||
onPartial: @escaping @Sendable (String) -> Void
|
||||
) async throws -> any CloudASRStreamingSession {
|
||||
_ = locale
|
||||
let credentials = try VolcengineCredentials.parse(
|
||||
apiKey: apiKey,
|
||||
fallbackResourceID: resolvedResourceID
|
||||
)
|
||||
let url = try resolvedEndpointURL()
|
||||
let connectID = UUID().uuidString
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.timeoutInterval = 8
|
||||
request.setValue(credentials.appID, forHTTPHeaderField: "X-Api-App-Key")
|
||||
request.setValue(credentials.accessToken, forHTTPHeaderField: "X-Api-Access-Key")
|
||||
request.setValue(credentials.resourceID, forHTTPHeaderField: "X-Api-Resource-Id")
|
||||
request.setValue(connectID, forHTTPHeaderField: "X-Api-Connect-Id")
|
||||
|
||||
let task = session.webSocketTask(with: request)
|
||||
task.resume()
|
||||
let live = VolcengineStreamingSession(
|
||||
wsTask: task,
|
||||
connectID: connectID,
|
||||
dictionary: dictionary,
|
||||
onPartial: onPartial
|
||||
)
|
||||
try await live.start()
|
||||
return live
|
||||
}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
_ = sampleRate
|
||||
guard !samples.isEmpty else { throw CloudASRError.emptyTranscript }
|
||||
let session = try await openStreamingSession(
|
||||
locale: locale,
|
||||
dictionary: dictionary,
|
||||
onPartial: { _ in }
|
||||
)
|
||||
try await session.append(samples: samples)
|
||||
let text = try await session.finish()
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { throw CloudASRError.emptyTranscript }
|
||||
return trimmed
|
||||
}
|
||||
|
||||
/// Settings connection probe: WebSocket upgrade + first-frame auth only.
|
||||
///
|
||||
/// Do not push silence through `transcribe` — SAUC returns an empty final
|
||||
/// then closes the socket, and `finish()` prefers that Socket error over
|
||||
/// `emptyTranscript`, so the default silence probe always failed while
|
||||
/// real dictation (with speech) still worked.
|
||||
func probeConnection() async throws {
|
||||
let live = try await openStreamingSession(
|
||||
locale: Locale(identifier: "zh-CN"),
|
||||
dictionary: .empty,
|
||||
onPartial: { _ in }
|
||||
)
|
||||
live.cancel()
|
||||
}
|
||||
|
||||
private var resolvedResourceID: String {
|
||||
resourceID.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? CloudASRModelCatalog.volcengineDefaultResourceID
|
||||
: resourceID.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
|
||||
private func resolvedEndpointURL() throws -> URL {
|
||||
let raw = endpoint.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? CloudASRModelCatalog.volcengineEndpoint
|
||||
: endpoint.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
|
||||
return url
|
||||
}
|
||||
|
||||
static func firstFramePayload(
|
||||
connectID: String,
|
||||
dictionary: PersonalDictionary
|
||||
) throws -> Data {
|
||||
var request: [String: Any] = [
|
||||
"model_name": "bigmodel",
|
||||
"enable_itn": true,
|
||||
"enable_punc": true,
|
||||
"show_utterances": true,
|
||||
"enable_speaker_info": true,
|
||||
// Official two-pass: stream interim for UI, nostream re-decode per
|
||||
// VAD sentence for definite polish-ready text (scheme A).
|
||||
"enable_nonstream": true,
|
||||
"end_window_size": 800,
|
||||
"force_to_speech_time": 1_000,
|
||||
]
|
||||
if let context = hotwordContext(dictionary: dictionary) {
|
||||
request["context"] = context
|
||||
}
|
||||
|
||||
let payload: [String: Any] = [
|
||||
"user": ["uid": connectID],
|
||||
"audio": [
|
||||
"format": "pcm",
|
||||
"rate": 16_000,
|
||||
"bits": 16,
|
||||
"channel": 1,
|
||||
"codec": "raw",
|
||||
],
|
||||
"request": request,
|
||||
]
|
||||
return try JSONSerialization.data(withJSONObject: payload)
|
||||
}
|
||||
|
||||
private static func hotwordContext(dictionary: PersonalDictionary) -> String? {
|
||||
var seen: [String] = []
|
||||
for word in dictionary.asrHotwords() {
|
||||
let trimmed = word.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { continue }
|
||||
guard !seen.contains(where: { $0.caseInsensitiveCompare(trimmed) == .orderedSame }) else {
|
||||
continue
|
||||
}
|
||||
seen.append(trimmed)
|
||||
if seen.count >= hotwordCap { break }
|
||||
}
|
||||
guard !seen.isEmpty else { return nil }
|
||||
let words = seen.map { ["word": $0] }
|
||||
guard let data = try? JSONSerialization.data(withJSONObject: ["hotwords": words]) else {
|
||||
return nil
|
||||
}
|
||||
return String(data: data, encoding: .utf8)
|
||||
}
|
||||
|
||||
static func displayText(from payload: Data) -> String {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: payload) as? [String: Any],
|
||||
let result = normalizedResult(from: json) else {
|
||||
return ""
|
||||
}
|
||||
|
||||
if let utterances = result["utterances"] as? [[String: Any]], !utterances.isEmpty {
|
||||
let pieces = utterances.compactMap { $0["text"] as? String }
|
||||
let joined = pieces.joined()
|
||||
if !joined.isEmpty { return joined }
|
||||
}
|
||||
return result["text"] as? String ?? ""
|
||||
}
|
||||
|
||||
/// Prefer definite (two-pass) utterance text for polish input.
|
||||
static func committedText(from payload: Data) -> String {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: payload) as? [String: Any],
|
||||
let result = normalizedResult(from: json),
|
||||
let utterances = result["utterances"] as? [[String: Any]],
|
||||
!utterances.isEmpty else {
|
||||
return ""
|
||||
}
|
||||
let definite = utterances.compactMap { utterance -> String? in
|
||||
let isDefinite = utterance["definite"] as? Bool ?? false
|
||||
guard isDefinite else { return nil }
|
||||
return utterance["text"] as? String
|
||||
}
|
||||
return definite.joined()
|
||||
}
|
||||
|
||||
private static func normalizedResult(from json: [String: Any]) -> [String: Any]? {
|
||||
if let result = json["result"] as? [String: Any] {
|
||||
return result
|
||||
}
|
||||
if let results = json["result"] as? [[String: Any]] {
|
||||
return results.first
|
||||
}
|
||||
if json["text"] as? String != nil {
|
||||
return json
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Utterance session
|
||||
|
||||
private final class VolcengineStreamingSession: CloudASRStreamingSession, @unchecked Sendable {
|
||||
private let wsTask: URLSessionWebSocketTask
|
||||
private let connectID: String
|
||||
private let dictionary: PersonalDictionary
|
||||
private let onPartial: @Sendable (String) -> Void
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
private var sequence: Int32 = 1
|
||||
private var pcmBuffer = Data()
|
||||
private var receiveTask: Task<Void, Never>?
|
||||
private var failure: Error?
|
||||
private var finished = false
|
||||
private var lastDisplay = ""
|
||||
private var lastCommitted = ""
|
||||
private var sawServerFinal = false
|
||||
|
||||
init(
|
||||
wsTask: URLSessionWebSocketTask,
|
||||
connectID: String,
|
||||
dictionary: PersonalDictionary,
|
||||
onPartial: @escaping @Sendable (String) -> Void
|
||||
) {
|
||||
self.wsTask = wsTask
|
||||
self.connectID = connectID
|
||||
self.dictionary = dictionary
|
||||
self.onPartial = onPartial
|
||||
}
|
||||
|
||||
func start() async throws {
|
||||
let firstPayload = try VolcengineCloudASRClient.firstFramePayload(
|
||||
connectID: connectID,
|
||||
dictionary: dictionary
|
||||
)
|
||||
try await send(
|
||||
VolcengineFrame.build(
|
||||
messageType: .fullClientRequest,
|
||||
flags: .positiveSequence,
|
||||
serialization: .json,
|
||||
payload: firstPayload,
|
||||
sequence: 1
|
||||
)
|
||||
)
|
||||
sequence = 2
|
||||
receiveTask = Task { [weak self] in
|
||||
await self?.receiveLoop()
|
||||
}
|
||||
}
|
||||
|
||||
func append(samples: [Float]) async throws {
|
||||
try throwIfFailed()
|
||||
let pcm = CloudASRStreamingPCM.pcm16LE(samples: samples)
|
||||
let (frames, nextSequences): ([Data], [Int32]) = lock.withLock {
|
||||
pcmBuffer.append(pcm)
|
||||
var frames: [Data] = []
|
||||
while pcmBuffer.count >= VolcengineCloudASRClient.targetChunkBytes {
|
||||
let frame = pcmBuffer.prefix(VolcengineCloudASRClient.targetChunkBytes)
|
||||
frames.append(Data(frame))
|
||||
pcmBuffer.removeFirst(VolcengineCloudASRClient.targetChunkBytes)
|
||||
}
|
||||
let nextSequences: [Int32] = frames.indices.map { _ in
|
||||
let seq = sequence
|
||||
sequence += 1
|
||||
return seq
|
||||
}
|
||||
return (frames, nextSequences)
|
||||
}
|
||||
|
||||
for (frame, seq) in zip(frames, nextSequences) {
|
||||
try await send(
|
||||
VolcengineFrame.build(
|
||||
messageType: .audioOnlyRequest,
|
||||
flags: .positiveSequence,
|
||||
serialization: .none,
|
||||
payload: frame,
|
||||
sequence: seq
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func finish() async throws -> String {
|
||||
try throwIfFailed()
|
||||
// Only consume a sequence number when we actually send trailing PCM.
|
||||
// Skipping an unused seq (common when length is an exact chunk multiple,
|
||||
// e.g. the settings probe's 1 s / 32_000-byte clip) makes the final
|
||||
// negative packet mismatch server autoAssignedSequence → error 45000000.
|
||||
let trailing = lock.withLock { () -> Data in
|
||||
let data = pcmBuffer
|
||||
pcmBuffer.removeAll(keepingCapacity: false)
|
||||
return data
|
||||
}
|
||||
|
||||
if !trailing.isEmpty {
|
||||
let endSequence = lock.withLock { () -> Int32 in
|
||||
let seq = sequence
|
||||
sequence += 1
|
||||
return seq
|
||||
}
|
||||
try await send(
|
||||
VolcengineFrame.build(
|
||||
messageType: .audioOnlyRequest,
|
||||
flags: .positiveSequence,
|
||||
serialization: .none,
|
||||
payload: trailing,
|
||||
sequence: endSequence
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
let negativeSeq = lock.withLock { () -> Int32 in
|
||||
let seq = sequence
|
||||
sequence += 1
|
||||
return seq
|
||||
}
|
||||
try await send(
|
||||
VolcengineFrame.build(
|
||||
messageType: .audioOnlyRequest,
|
||||
flags: .negativeSequence,
|
||||
serialization: .none,
|
||||
payload: Data(),
|
||||
sequence: -negativeSeq
|
||||
)
|
||||
)
|
||||
|
||||
let deadline = Date().addingTimeInterval(VolcengineCloudASRClient.finalTimeout)
|
||||
while Date() < deadline {
|
||||
try throwIfFailed()
|
||||
let snapshot = lock.withLock { (sawServerFinal, lastCommitted, lastDisplay) }
|
||||
if snapshot.0 {
|
||||
let text = snapshot.1.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? snapshot.2
|
||||
: snapshot.1
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
cancel()
|
||||
if trimmed.isEmpty { throw CloudASRError.emptyTranscript }
|
||||
return trimmed
|
||||
}
|
||||
try await Task.sleep(nanoseconds: 20_000_000)
|
||||
}
|
||||
cancel()
|
||||
throw CloudASRError.transport("Volcengine final result timed out")
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
receiveTask?.cancel()
|
||||
wsTask.cancel(with: .normalClosure, reason: nil)
|
||||
lock.withLock { finished = true }
|
||||
}
|
||||
|
||||
private func receiveLoop() async {
|
||||
while !Task.isCancelled {
|
||||
let message: URLSessionWebSocketTask.Message
|
||||
do {
|
||||
message = try await wsTask.receive()
|
||||
} catch {
|
||||
publishFailure(CloudASRError.transport(error.localizedDescription))
|
||||
return
|
||||
}
|
||||
|
||||
let data: Data
|
||||
switch message {
|
||||
case .data(let payload):
|
||||
data = payload
|
||||
case .string(let string):
|
||||
data = Data(string.utf8)
|
||||
@unknown default:
|
||||
continue
|
||||
}
|
||||
|
||||
guard let frame = VolcengineFrame.parse(data) else { continue }
|
||||
if frame.messageType == .errorMessage {
|
||||
let body = String(data: frame.payload, encoding: .utf8) ?? ""
|
||||
let code = frame.errorCode ?? 0
|
||||
publishFailure(CloudASRError.transport("ASR error \(code): \(body)"))
|
||||
return
|
||||
}
|
||||
guard frame.messageType == .fullServerResponse else { continue }
|
||||
|
||||
let display = VolcengineCloudASRClient.displayText(from: frame.payload)
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let committed = VolcengineCloudASRClient.committedText(from: frame.payload)
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
|
||||
let emit = lock.withLock { () -> String in
|
||||
if !display.isEmpty {
|
||||
lastDisplay = display
|
||||
}
|
||||
if !committed.isEmpty {
|
||||
lastCommitted = committed
|
||||
}
|
||||
if frame.isFinal {
|
||||
sawServerFinal = true
|
||||
}
|
||||
return lastDisplay
|
||||
}
|
||||
|
||||
if !emit.isEmpty {
|
||||
onPartial(emit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func send(_ data: Data) async throws {
|
||||
do {
|
||||
try await wsTask.send(.data(data))
|
||||
} catch {
|
||||
throw CloudASRError.transport(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private func throwIfFailed() throws {
|
||||
let (error, done) = lock.withLock { (failure, finished) }
|
||||
if let error { throw error }
|
||||
if done { throw CloudASRError.transport("Volcengine session cancelled") }
|
||||
}
|
||||
|
||||
private func publishFailure(_ error: Error) {
|
||||
lock.withLock { failure = error }
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
private struct VolcengineCredentials {
|
||||
let appID: String
|
||||
let accessToken: String
|
||||
let resourceID: String
|
||||
|
||||
static func parse(apiKey: String, fallbackResourceID: String) throws -> VolcengineCredentials {
|
||||
let trimmed = apiKey.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
|
||||
if let data = trimmed.data(using: .utf8),
|
||||
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {
|
||||
let appID = string(json, keys: ["app_id", "appId", "appid"])
|
||||
let token = string(json, keys: ["access_token", "accessToken", "token"])
|
||||
let resourceID = string(json, keys: ["resource_id", "resourceId", "resource"])
|
||||
?? fallbackResourceID
|
||||
guard let appID, let token, !resourceID.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
return VolcengineCredentials(appID: appID, accessToken: token, resourceID: resourceID)
|
||||
}
|
||||
|
||||
let separators = CharacterSet(charactersIn: ":\n,")
|
||||
let parts = trimmed
|
||||
.components(separatedBy: separators)
|
||||
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
|
||||
.filter { !$0.isEmpty }
|
||||
guard parts.count >= 2 else { throw CloudASRError.noAPIKey }
|
||||
let resourceID = parts.count >= 3 ? parts[2] : fallbackResourceID
|
||||
return VolcengineCredentials(appID: parts[0], accessToken: parts[1], resourceID: resourceID)
|
||||
}
|
||||
|
||||
private static func string(_ json: [String: Any], keys: [String]) -> String? {
|
||||
for key in keys {
|
||||
if let value = json[key] as? String {
|
||||
let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !trimmed.isEmpty { return trimmed }
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
enum VolcengineMessageType: UInt8 {
|
||||
case fullClientRequest = 0b0001
|
||||
case audioOnlyRequest = 0b0010
|
||||
case fullServerResponse = 0b1001
|
||||
case errorMessage = 0b1111
|
||||
}
|
||||
|
||||
enum VolcengineFlags: UInt8 {
|
||||
case none = 0b0000
|
||||
case positiveSequence = 0b0001
|
||||
case lastPacket = 0b0010
|
||||
case negativeSequence = 0b0011
|
||||
}
|
||||
|
||||
enum VolcengineSerialization: UInt8 {
|
||||
case none = 0b0000
|
||||
case json = 0b0001
|
||||
}
|
||||
|
||||
struct VolcengineFrame {
|
||||
let messageType: VolcengineMessageType?
|
||||
let flags: UInt8
|
||||
let sequence: Int32?
|
||||
let errorCode: UInt32?
|
||||
let payload: Data
|
||||
|
||||
var isFinal: Bool {
|
||||
flags == VolcengineFlags.lastPacket.rawValue
|
||||
|| flags == VolcengineFlags.negativeSequence.rawValue
|
||||
|| (sequence ?? 0) < 0
|
||||
}
|
||||
|
||||
static func build(
|
||||
messageType: VolcengineMessageType,
|
||||
flags: VolcengineFlags,
|
||||
serialization: VolcengineSerialization,
|
||||
payload: Data,
|
||||
sequence: Int32?
|
||||
) -> Data {
|
||||
var data = Data()
|
||||
data.append(0x11)
|
||||
data.append((messageType.rawValue << 4) | flags.rawValue)
|
||||
data.append(serialization.rawValue << 4)
|
||||
data.append(0x00)
|
||||
|
||||
if flags == .positiveSequence || flags == .negativeSequence, let sequence {
|
||||
data.appendBE32(UInt32(bitPattern: sequence))
|
||||
}
|
||||
data.appendBE32(UInt32(payload.count))
|
||||
data.append(payload)
|
||||
return data
|
||||
}
|
||||
|
||||
static func parse(_ data: Data) -> VolcengineFrame? {
|
||||
guard data.count >= 8 else { return nil }
|
||||
let bytes = [UInt8](data)
|
||||
let headerSize = Int(bytes[0] & 0x0F) * 4
|
||||
guard headerSize >= 4, data.count >= headerSize + 4 else { return nil }
|
||||
|
||||
let typeRaw = (bytes[1] >> 4) & 0x0F
|
||||
let messageType = VolcengineMessageType(rawValue: typeRaw)
|
||||
let flags = bytes[1] & 0x0F
|
||||
let compression = bytes[2] & 0x0F
|
||||
guard compression == 0 else { return nil }
|
||||
|
||||
var offset = headerSize
|
||||
var sequence: Int32?
|
||||
if flags == VolcengineFlags.positiveSequence.rawValue
|
||||
|| flags == VolcengineFlags.negativeSequence.rawValue {
|
||||
guard let value = data.readBE32(at: offset) else { return nil }
|
||||
sequence = Int32(bitPattern: value)
|
||||
offset += 4
|
||||
}
|
||||
|
||||
if messageType == .errorMessage {
|
||||
guard let code = data.readBE32(at: offset),
|
||||
let size = data.readBE32(at: offset + 4) else { return nil }
|
||||
offset += 8
|
||||
guard data.count >= offset + Int(size) else { return nil }
|
||||
return VolcengineFrame(
|
||||
messageType: messageType,
|
||||
flags: flags,
|
||||
sequence: sequence,
|
||||
errorCode: code,
|
||||
payload: data.subdata(in: offset..<(offset + Int(size)))
|
||||
)
|
||||
}
|
||||
|
||||
guard let size = data.readBE32(at: offset) else { return nil }
|
||||
offset += 4
|
||||
guard data.count >= offset + Int(size) else { return nil }
|
||||
return VolcengineFrame(
|
||||
messageType: messageType,
|
||||
flags: flags,
|
||||
sequence: sequence,
|
||||
errorCode: nil,
|
||||
payload: data.subdata(in: offset..<(offset + Int(size)))
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private extension Data {
|
||||
mutating func appendBE32(_ value: UInt32) {
|
||||
var bigEndian = value.bigEndian
|
||||
Swift.withUnsafeBytes(of: &bigEndian) { append(contentsOf: $0) }
|
||||
}
|
||||
|
||||
func readBE32(at offset: Int) -> UInt32? {
|
||||
guard count >= offset + 4 else { return nil }
|
||||
return self[offset..<(offset + 4)].reduce(UInt32(0)) { ($0 << 8) | UInt32($1) }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,449 @@
|
||||
// CustomLanguageModelManager.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Prepares the bundled SFCustomLanguageModelData asset on device and shares
|
||||
// the compiled LM + Vocab through the App Group container. Both the host app
|
||||
// and keyboard extension read the same prepared configuration for
|
||||
// DictationTranscriber content hints.
|
||||
|
||||
import Foundation
|
||||
import Speech
|
||||
import os
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public final class CustomLanguageModelManager: @unchecked Sendable {
|
||||
|
||||
public static let shared = CustomLanguageModelManager()
|
||||
|
||||
public enum PrepareState: Equatable, Sendable {
|
||||
case idle
|
||||
case preparing
|
||||
case ready
|
||||
case failed(String)
|
||||
}
|
||||
|
||||
struct BundledManifest: Decodable {
|
||||
let version: String
|
||||
let bin_bytes: Int
|
||||
let identifier: String
|
||||
}
|
||||
|
||||
private enum Storage {
|
||||
static let subdirectory = "CustomLanguageModel/v1"
|
||||
static let fingerprintKey = "customLM.preparedFingerprint"
|
||||
static let preparedAtKey = "customLM.preparedAt"
|
||||
static let lastFailureAtKey = "customLM.lastFailureAt"
|
||||
static let attemptCountKey = "customLM.attemptCount"
|
||||
static let maxRetryAttempts = 3
|
||||
/// Backoff after failure attempts 1, 2, and 3 (seconds).
|
||||
static let backoffIntervals: [TimeInterval] = [30, 120, 600]
|
||||
}
|
||||
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
private var cachedConfiguration: SFSpeechLanguageModel.Configuration?
|
||||
private var state: PrepareState = .idle
|
||||
private var prepareTask: Task<Void, Never>?
|
||||
|
||||
private init() {}
|
||||
|
||||
// MARK: - Public API
|
||||
|
||||
/// Returns a prepared configuration for Chinese locales when available.
|
||||
public func configurationForTranscription(locale: Locale) -> SFSpeechLanguageModel.Configuration? {
|
||||
guard Self.isChineseLocale(locale) else { return nil }
|
||||
return lock.withLock { () -> SFSpeechLanguageModel.Configuration? in
|
||||
if let cachedConfiguration {
|
||||
return cachedConfiguration
|
||||
}
|
||||
if let loaded = Self.loadCachedConfigurationFromDisk() {
|
||||
cachedConfiguration = loaded
|
||||
state = .ready
|
||||
return loaded
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
public func currentState() -> PrepareState {
|
||||
lock.withLock { state }
|
||||
}
|
||||
|
||||
/// Fire-and-forget preparation for the host app. Safe to call repeatedly.
|
||||
/// Retries after exponential backoff when a prior attempt failed.
|
||||
public func prepareInBackgroundIfNeeded() {
|
||||
#if os(iOS)
|
||||
guard AppGroup.isAvailable else { return }
|
||||
#endif
|
||||
|
||||
let shouldStart = lock.withLock { () -> Bool in
|
||||
if case .preparing = state { return false }
|
||||
if cachedConfiguration != nil { return false }
|
||||
if let loaded = Self.loadCachedConfigurationFromDisk() {
|
||||
cachedConfiguration = loaded
|
||||
state = .ready
|
||||
Self.clearRetryState()
|
||||
return false
|
||||
}
|
||||
if prepareTask != nil { return false }
|
||||
|
||||
if case .failed = state {
|
||||
guard Self.canRetryAfterFailure() else { return false }
|
||||
} else if !Self.canRetryAfterFailure() {
|
||||
return false
|
||||
}
|
||||
|
||||
state = .preparing
|
||||
return true
|
||||
}
|
||||
guard shouldStart else { return }
|
||||
|
||||
prepareTask = Task.detached(priority: .utility) { [weak self] in
|
||||
guard let self else { return }
|
||||
defer {
|
||||
self.lock.withLock { self.prepareTask = nil }
|
||||
}
|
||||
do {
|
||||
_ = try await self.prepareIfNeeded()
|
||||
} catch {
|
||||
Self.recordFailure()
|
||||
self.lock.withLock {
|
||||
self.state = .failed(error.localizedDescription)
|
||||
}
|
||||
Self.log(
|
||||
"prepare failed (attempt \(Self.storedAttemptCount())): \(error.localizedDescription)"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Prepares the bundled training asset into the App Group container.
|
||||
@discardableResult
|
||||
public func prepareIfNeeded() async throws -> SFSpeechLanguageModel.Configuration? {
|
||||
if let existing = configurationForTranscription(locale: Locale(identifier: "zh-Hans")) {
|
||||
lock.withLock { state = .ready }
|
||||
Self.clearRetryState()
|
||||
return existing
|
||||
}
|
||||
|
||||
guard Self.canRetryAfterFailure() else {
|
||||
throw PrepareError.retryBudgetExhausted
|
||||
}
|
||||
|
||||
guard let manifest = Self.bundledManifest() else {
|
||||
throw PrepareError.missingManifest
|
||||
}
|
||||
guard let assetURL = Self.bundledTrainingAssetURL() else {
|
||||
throw PrepareError.missingTrainingAsset
|
||||
}
|
||||
guard let preparedDir = Self.preparedDirectoryURL() else {
|
||||
throw PrepareError.missingAppGroupContainer
|
||||
}
|
||||
|
||||
let fingerprint = Self.fingerprint(for: manifest)
|
||||
if Self.storedFingerprint() == fingerprint,
|
||||
let cached = Self.loadCachedConfigurationFromDisk() {
|
||||
lock.withLock {
|
||||
cachedConfiguration = cached
|
||||
state = .ready
|
||||
}
|
||||
Self.clearRetryState()
|
||||
return cached
|
||||
}
|
||||
|
||||
lock.withLock { state = .preparing }
|
||||
|
||||
let languageModelURL = preparedDir.appendingPathComponent("LM")
|
||||
let vocabularyURL = preparedDir.appendingPathComponent("Vocab")
|
||||
try Self.removeItemIfExists(at: languageModelURL)
|
||||
try Self.removeItemIfExists(at: vocabularyURL)
|
||||
|
||||
let configuration = SFSpeechLanguageModel.Configuration(
|
||||
languageModel: languageModelURL,
|
||||
vocabulary: vocabularyURL
|
||||
)
|
||||
|
||||
Self.log(
|
||||
"preparing custom LM (\(manifest.bin_bytes) byte asset)… \(OSGDiag.memoryTag())"
|
||||
)
|
||||
OSGDiag.log(
|
||||
"clm.prepare.begin bytes=\(manifest.bin_bytes) \(OSGDiag.memoryTag())",
|
||||
category: "asr"
|
||||
)
|
||||
try await Self.prepareLanguageModel(assetURL: assetURL, configuration: configuration)
|
||||
|
||||
guard FileManager.default.fileExists(atPath: languageModelURL.path),
|
||||
FileManager.default.fileExists(atPath: vocabularyURL.path) else {
|
||||
throw PrepareError.missingPreparedArtifacts
|
||||
}
|
||||
|
||||
Self.persistenceDefaults.set(fingerprint, forKey: Storage.fingerprintKey)
|
||||
Self.persistenceDefaults.set(Date().timeIntervalSince1970, forKey: Storage.preparedAtKey)
|
||||
Self.clearRetryState()
|
||||
|
||||
lock.withLock {
|
||||
cachedConfiguration = configuration
|
||||
state = .ready
|
||||
}
|
||||
|
||||
Self.log("custom LM ready at \(preparedDir.path)")
|
||||
OSGDiag.log(
|
||||
"clm.prepare.done path=\(preparedDir.lastPathComponent) \(OSGDiag.memoryTag())",
|
||||
category: "asr"
|
||||
)
|
||||
return configuration
|
||||
}
|
||||
|
||||
// MARK: - DictationTranscriber factory (iOS host app)
|
||||
|
||||
#if os(iOS)
|
||||
public static func makeDictationTranscriber(
|
||||
locale: Locale,
|
||||
lmConfiguration: SFSpeechLanguageModel.Configuration?
|
||||
) -> DictationTranscriber {
|
||||
let preset = DictationTranscriber.Preset.progressiveLongDictation
|
||||
guard let lmConfiguration, isChineseLocale(locale) else {
|
||||
return DictationTranscriber(locale: locale, preset: preset)
|
||||
}
|
||||
|
||||
let contentHints = preset.contentHints.union([
|
||||
.customizedLanguage(modelConfiguration: lmConfiguration),
|
||||
])
|
||||
return DictationTranscriber(
|
||||
locale: locale,
|
||||
contentHints: contentHints,
|
||||
transcriptionOptions: preset.transcriptionOptions,
|
||||
reportingOptions: preset.reportingOptions,
|
||||
attributeOptions: preset.attributeOptions
|
||||
)
|
||||
}
|
||||
#endif
|
||||
|
||||
// MARK: - Legacy Speech request (macOS Apple Speech fallback)
|
||||
|
||||
/// Up to 100 short phrases for `SFSpeechRecognitionRequest.contextualStrings`.
|
||||
public static func contextualStringsForRecognition(
|
||||
bias: LocalASRBiasPayload?,
|
||||
maxCount: Int = 100
|
||||
) -> [String] {
|
||||
guard let bias, !bias.hardHotwords.isEmpty else { return [] }
|
||||
return Array(bias.hardHotwords.prefix(max(1, maxCount)))
|
||||
}
|
||||
|
||||
/// Applies bundled CLM + optional contextual strings to a legacy on-device request.
|
||||
public static func applyCustomLanguageModel(
|
||||
to request: SFSpeechURLRecognitionRequest,
|
||||
locale: Locale,
|
||||
bias: LocalASRBiasPayload?
|
||||
) {
|
||||
request.requiresOnDeviceRecognition = true
|
||||
if let configuration = shared.configurationForTranscription(locale: locale) {
|
||||
request.customizedLanguageModel = configuration
|
||||
}
|
||||
let phrases = contextualStringsForRecognition(bias: bias)
|
||||
if !phrases.isEmpty {
|
||||
request.contextualStrings = phrases
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Bundle / disk helpers
|
||||
|
||||
private static var resourceBundle: Bundle {
|
||||
// CLM assets ship in the host app bundle (not the extension Shared
|
||||
// framework) so the keyboard process never mmaps the training bin.
|
||||
Bundle.main
|
||||
}
|
||||
|
||||
static func bundledTrainingAssetURL() -> URL? {
|
||||
// Prefer flat Bundle.main (iOS HostCLM copy), then legacy subdirs.
|
||||
let candidates: [URL?] = [
|
||||
resourceBundle.url(forResource: "OSGKeyboardCLM", withExtension: "bin"),
|
||||
resourceBundle.url(
|
||||
forResource: "OSGKeyboardCLM",
|
||||
withExtension: "bin",
|
||||
subdirectory: "HostCLM/v1"
|
||||
),
|
||||
resourceBundle.url(
|
||||
forResource: "OSGKeyboardCLM",
|
||||
withExtension: "bin",
|
||||
subdirectory: Storage.subdirectory
|
||||
),
|
||||
]
|
||||
return candidates.compactMap { $0 }.first
|
||||
}
|
||||
|
||||
static func bundledManifest() -> BundledManifest? {
|
||||
let candidates: [URL?] = [
|
||||
resourceBundle.url(forResource: "compiled-manifest", withExtension: "json"),
|
||||
resourceBundle.url(
|
||||
forResource: "compiled-manifest",
|
||||
withExtension: "json",
|
||||
subdirectory: "HostCLM/v1"
|
||||
),
|
||||
resourceBundle.url(
|
||||
forResource: "compiled-manifest",
|
||||
withExtension: "json",
|
||||
subdirectory: Storage.subdirectory
|
||||
),
|
||||
]
|
||||
guard let manifestURL = candidates.compactMap({ $0 }).first,
|
||||
let data = try? Data(contentsOf: manifestURL),
|
||||
let manifest = try? JSONDecoder().decode(BundledManifest.self, from: data)
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
return manifest
|
||||
}
|
||||
|
||||
static func preparedDirectoryURL() -> URL? {
|
||||
if let container = FileManager.default.containerURL(
|
||||
forSecurityApplicationGroupIdentifier: AppGroup.identifier
|
||||
) {
|
||||
let directory = container.appendingPathComponent(Storage.subdirectory, isDirectory: true)
|
||||
try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
||||
return directory
|
||||
}
|
||||
#if os(macOS)
|
||||
guard let appSupport = FileManager.default.urls(
|
||||
for: .applicationSupportDirectory,
|
||||
in: .userDomainMask
|
||||
).first else {
|
||||
return nil
|
||||
}
|
||||
let directory = appSupport
|
||||
.appendingPathComponent("OSGKeyboard", isDirectory: true)
|
||||
.appendingPathComponent(Storage.subdirectory, isDirectory: true)
|
||||
try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
||||
return directory
|
||||
#else
|
||||
return nil
|
||||
#endif
|
||||
}
|
||||
|
||||
static func loadCachedConfigurationFromDisk() -> SFSpeechLanguageModel.Configuration? {
|
||||
guard let manifest = bundledManifest(),
|
||||
storedFingerprint() == fingerprint(for: manifest),
|
||||
let preparedDir = preparedDirectoryURL()
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
|
||||
let languageModelURL = preparedDir.appendingPathComponent("LM")
|
||||
let vocabularyURL = preparedDir.appendingPathComponent("Vocab")
|
||||
let fm = FileManager.default
|
||||
guard fm.fileExists(atPath: languageModelURL.path),
|
||||
fm.fileExists(atPath: vocabularyURL.path) else {
|
||||
return nil
|
||||
}
|
||||
|
||||
return SFSpeechLanguageModel.Configuration(
|
||||
languageModel: languageModelURL,
|
||||
vocabulary: vocabularyURL
|
||||
)
|
||||
}
|
||||
|
||||
static func isChineseLocale(_ locale: Locale) -> Bool {
|
||||
locale.identifier(.bcp47).lowercased().hasPrefix("zh")
|
||||
}
|
||||
|
||||
private static func fingerprint(for manifest: BundledManifest) -> String {
|
||||
"\(manifest.identifier)|\(manifest.version)|\(manifest.bin_bytes)"
|
||||
}
|
||||
|
||||
private static func storedFingerprint() -> String? {
|
||||
persistenceDefaults.string(forKey: Storage.fingerprintKey)
|
||||
}
|
||||
|
||||
private static func removeItemIfExists(at url: URL) throws {
|
||||
let fm = FileManager.default
|
||||
if fm.fileExists(atPath: url.path) {
|
||||
try fm.removeItem(at: url)
|
||||
}
|
||||
}
|
||||
|
||||
private static func prepareLanguageModel(
|
||||
assetURL: URL,
|
||||
configuration: SFSpeechLanguageModel.Configuration
|
||||
) async throws {
|
||||
try await withCheckedThrowingContinuation {
|
||||
(continuation: CheckedContinuation<Void, Error>) in
|
||||
SFSpeechLanguageModel.prepareCustomLanguageModel(
|
||||
for: assetURL,
|
||||
configuration: configuration
|
||||
) { error in
|
||||
if let error {
|
||||
continuation.resume(throwing: error)
|
||||
} else {
|
||||
continuation.resume()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Retry / backoff
|
||||
|
||||
private static var persistenceDefaults: UserDefaults {
|
||||
AppGroup.defaultsIfAvailable ?? .standard
|
||||
}
|
||||
|
||||
private static func storedAttemptCount() -> Int {
|
||||
persistenceDefaults.integer(forKey: Storage.attemptCountKey)
|
||||
}
|
||||
|
||||
private static func storedLastFailureAt() -> TimeInterval? {
|
||||
let value = persistenceDefaults.double(forKey: Storage.lastFailureAtKey)
|
||||
return value > 0 ? value : nil
|
||||
}
|
||||
|
||||
private static func recordFailure() {
|
||||
let nextAttempt = storedAttemptCount() + 1
|
||||
persistenceDefaults.set(nextAttempt, forKey: Storage.attemptCountKey)
|
||||
persistenceDefaults.set(Date().timeIntervalSince1970, forKey: Storage.lastFailureAtKey)
|
||||
}
|
||||
|
||||
private static func clearRetryState() {
|
||||
persistenceDefaults.removeObject(forKey: Storage.attemptCountKey)
|
||||
persistenceDefaults.removeObject(forKey: Storage.lastFailureAtKey)
|
||||
}
|
||||
|
||||
/// Returns false when retry budget is exhausted or backoff has not elapsed.
|
||||
private static func canRetryAfterFailure() -> Bool {
|
||||
let attempts = storedAttemptCount()
|
||||
guard attempts > 0 else { return true }
|
||||
guard attempts <= Storage.maxRetryAttempts else { return false }
|
||||
|
||||
guard let lastFailureAt = storedLastFailureAt() else { return true }
|
||||
let backoffIndex = min(attempts - 1, Storage.backoffIntervals.count - 1)
|
||||
let requiredDelay = Storage.backoffIntervals[backoffIndex]
|
||||
let elapsed = Date().timeIntervalSince1970 - lastFailureAt
|
||||
return elapsed >= requiredDelay
|
||||
}
|
||||
|
||||
private static func log(_ message: String) {
|
||||
OSGLog.clm.info("\(message, privacy: .public)")
|
||||
}
|
||||
|
||||
enum PrepareError: LocalizedError {
|
||||
case missingManifest
|
||||
case missingTrainingAsset
|
||||
case missingAppGroupContainer
|
||||
case missingPreparedArtifacts
|
||||
case retryBudgetExhausted
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .missingManifest:
|
||||
return "Missing bundled custom language model manifest."
|
||||
case .missingTrainingAsset:
|
||||
return "Missing bundled custom language model training asset."
|
||||
case .missingAppGroupContainer:
|
||||
return "App Group container unavailable for custom language model preparation."
|
||||
case .missingPreparedArtifacts:
|
||||
return "Custom language model preparation did not produce LM/Vocab artifacts."
|
||||
case .retryBudgetExhausted:
|
||||
return "Custom language model preparation retry budget exhausted."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
// FlowAudioSessionCoordinator.swift
|
||||
// OSGKeyboard · Host Support
|
||||
//
|
||||
// Single owner for the shared iOS AVAudioSession used by Flow capture and
|
||||
// Picture in Picture keep-alive. Keeping category transitions here prevents
|
||||
// two independent state machines from racing the same process-wide session.
|
||||
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
import OSGKeyboardShared
|
||||
|
||||
public enum FlowAudioRouteRecoveryPolicy {
|
||||
public static func shouldRebuild(
|
||||
reasonRaw: UInt,
|
||||
formatIsStable: Bool,
|
||||
engineIsRunning: Bool
|
||||
) -> Bool {
|
||||
switch AVAudioSession.RouteChangeReason(rawValue: reasonRaw) {
|
||||
case .newDeviceAvailable, .oldDeviceUnavailable:
|
||||
return true
|
||||
case .categoryChange, .routeConfigurationChange:
|
||||
return !formatIsStable || !engineIsRunning
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public struct FlowAudioSessionSnapshot: Sendable, Equatable {
|
||||
public let sampleRate: Double
|
||||
public let inputChannels: Int
|
||||
public let inputPortType: String
|
||||
public let inputPortUID: String
|
||||
}
|
||||
|
||||
public final class FlowAudioEngineHandle: @unchecked Sendable {
|
||||
public let engine: AVAudioEngine
|
||||
|
||||
public init(_ engine: AVAudioEngine) {
|
||||
self.engine = engine
|
||||
}
|
||||
}
|
||||
|
||||
public final class FlowAudioSessionCoordinator: @unchecked Sendable {
|
||||
private struct CaptureActivation: Sendable {
|
||||
let snapshot: FlowAudioSessionSnapshot
|
||||
let preferredInputUID: String?
|
||||
}
|
||||
|
||||
public enum Mode: Sendable, Equatable {
|
||||
case inactive
|
||||
case playback
|
||||
case capture
|
||||
}
|
||||
|
||||
public static let shared = FlowAudioSessionCoordinator()
|
||||
|
||||
private let session = AVAudioSession.sharedInstance()
|
||||
private let queue = DispatchQueue(
|
||||
label: "com.osgkeyboard.flow.audio-session",
|
||||
qos: .userInitiated
|
||||
)
|
||||
private var mode: Mode = .inactive
|
||||
private var active = false
|
||||
|
||||
private init() {}
|
||||
|
||||
public func activateCapture() async throws -> FlowAudioSessionSnapshot {
|
||||
let activation: CaptureActivation = try await perform {
|
||||
if self.mode != .capture {
|
||||
try self.session.setCategory(
|
||||
.playAndRecord,
|
||||
mode: .measurement,
|
||||
options: [.defaultToSpeaker, .allowBluetoothHFP, .mixWithOthers]
|
||||
)
|
||||
}
|
||||
if !self.active {
|
||||
try self.session.setActive(true, options: .notifyOthersOnDeactivation)
|
||||
self.active = true
|
||||
}
|
||||
// When an HFP device is already connected, iOS can initially
|
||||
// report the built-in mic and switch to HFP after recording has
|
||||
// begun. Select the route before building AVAudioEngine so short
|
||||
// utterances do not land entirely inside that negotiation window.
|
||||
let hfp = self.session.availableInputs?.first(where: {
|
||||
$0.portType == .bluetoothHFP
|
||||
})
|
||||
if self.session.currentRoute.inputs.first?.portType != .bluetoothHFP,
|
||||
let hfp {
|
||||
do {
|
||||
try self.session.setPreferredInput(hfp)
|
||||
} catch {
|
||||
OSGDiag.log(
|
||||
"preferred HFP input failed: \(error.localizedDescription)",
|
||||
category: "flow"
|
||||
)
|
||||
}
|
||||
}
|
||||
self.mode = .capture
|
||||
return CaptureActivation(
|
||||
snapshot: self.makeSnapshot(),
|
||||
preferredInputUID: hfp?.uid
|
||||
)
|
||||
}
|
||||
return await waitForStableRoute(
|
||||
initial: activation.snapshot,
|
||||
preferredInputUID: activation.preferredInputUID
|
||||
)
|
||||
}
|
||||
|
||||
public func activatePlayback() async -> Bool {
|
||||
do {
|
||||
_ = try await perform {
|
||||
// Once capture has established a stable playAndRecord route,
|
||||
// keep that active session between utterances. The engine and
|
||||
// tap are stopped separately, so the microphone is released,
|
||||
// while avoiding a category flip that eventually yields !pri
|
||||
// after repeated PiP/capture cycles.
|
||||
if self.active, self.mode == .capture {
|
||||
return self.makeSnapshot()
|
||||
}
|
||||
if self.mode != .playback {
|
||||
try self.session.setCategory(
|
||||
.playback,
|
||||
mode: .moviePlayback,
|
||||
options: [.mixWithOthers]
|
||||
)
|
||||
}
|
||||
if !self.active {
|
||||
try self.session.setActive(true)
|
||||
self.active = true
|
||||
}
|
||||
self.mode = .playback
|
||||
return self.makeSnapshot()
|
||||
}
|
||||
return true
|
||||
} catch {
|
||||
OSGDiag.log(
|
||||
"PiP audio session failed: \(error.localizedDescription)",
|
||||
category: "flow"
|
||||
)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
public func deactivate() {
|
||||
queue.async { [self] in
|
||||
guard active else { return }
|
||||
do {
|
||||
try session.setActive(false, options: .notifyOthersOnDeactivation)
|
||||
active = false
|
||||
mode = .inactive
|
||||
} catch {
|
||||
OSGDiag.log(
|
||||
"audio session deactivate failed: \(error.localizedDescription)",
|
||||
category: "flow"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public func startEngine(_ handle: FlowAudioEngineHandle) async throws {
|
||||
try await perform {
|
||||
handle.engine.prepare()
|
||||
try handle.engine.start()
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
public func stopEngine(_ handle: FlowAudioEngineHandle) async {
|
||||
_ = try? await perform {
|
||||
if handle.engine.isRunning {
|
||||
handle.engine.stop()
|
||||
}
|
||||
handle.engine.reset()
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
public func enqueueStopEngine(_ handle: FlowAudioEngineHandle) {
|
||||
queue.async {
|
||||
if handle.engine.isRunning {
|
||||
handle.engine.stop()
|
||||
}
|
||||
handle.engine.reset()
|
||||
}
|
||||
}
|
||||
|
||||
private func waitForStableRoute(
|
||||
initial: FlowAudioSessionSnapshot,
|
||||
preferredInputUID: String?,
|
||||
timeout: TimeInterval = 1.5
|
||||
) async -> FlowAudioSessionSnapshot {
|
||||
let deadline = Date().addingTimeInterval(timeout)
|
||||
var previous = initial
|
||||
var stableReadCount = 0
|
||||
while Date() < deadline {
|
||||
try? await Task.sleep(nanoseconds: 50_000_000)
|
||||
let current = await snapshot()
|
||||
let preferredRouteReached = preferredInputUID == nil
|
||||
|| current.inputPortUID == preferredInputUID
|
||||
if current == previous, current.sampleRate > 0,
|
||||
current.inputChannels > 0, preferredRouteReached {
|
||||
stableReadCount += 1
|
||||
} else {
|
||||
stableReadCount = 0
|
||||
}
|
||||
if stableReadCount >= 4 {
|
||||
return current
|
||||
}
|
||||
previous = current
|
||||
}
|
||||
return previous
|
||||
}
|
||||
|
||||
private func snapshot() async -> FlowAudioSessionSnapshot {
|
||||
(try? await perform { self.makeSnapshot() })
|
||||
?? FlowAudioSessionSnapshot(
|
||||
sampleRate: 0,
|
||||
inputChannels: 0,
|
||||
inputPortType: "none",
|
||||
inputPortUID: ""
|
||||
)
|
||||
}
|
||||
|
||||
private func makeSnapshot() -> FlowAudioSessionSnapshot {
|
||||
let input = session.currentRoute.inputs.first
|
||||
return FlowAudioSessionSnapshot(
|
||||
sampleRate: session.sampleRate,
|
||||
inputChannels: session.inputNumberOfChannels,
|
||||
inputPortType: input?.portType.rawValue ?? "none",
|
||||
inputPortUID: input?.uid ?? ""
|
||||
)
|
||||
}
|
||||
|
||||
private func perform<T: Sendable>(
|
||||
_ operation: @escaping @Sendable () throws -> T
|
||||
) async throws -> T {
|
||||
try await withCheckedThrowingContinuation { continuation in
|
||||
queue.async {
|
||||
continuation.resume(with: Result { try operation() })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,612 @@
|
||||
// LiveDictationController.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Unified on-device dictation session: mic capture + iOS 26 SpeechAnalyzer.
|
||||
// Used by the keyboard preview sheet, host-app dictation handoff, and any
|
||||
// other foreground surface that needs live ASR without duplicating pipeline code.
|
||||
//
|
||||
// STATUS (v0.1.2): Retained as a "preview / one-shot handoff" path.
|
||||
// The *primary* voice-session path is `FlowSessionManager` +
|
||||
// `FlowContinuousCapture` (TypeWhisper-style continuous capture shared
|
||||
// between host app and keyboard extension). The keyboard extension
|
||||
// consumes results through `FlowSessionBridge`.
|
||||
//
|
||||
// This class is still imported by:
|
||||
// - `OSGKeyboard/Views/PreviewASRController.swift` (typealias)
|
||||
// - `OSGKeyboard/Views/KeyboardPreviewSheet.swift` (in-app preview)
|
||||
// - `OSGKeyboard/Views/KeyboardPreviewSheet.swift` (host-app ASR preview)
|
||||
// - `OSGKeyboardTests/PreviewASRControllerStateTests.swift`
|
||||
//
|
||||
// Do NOT remove without updating those call sites. The earlier
|
||||
// `OSGKeyboardExt/Services/AudioCaptureService.swift` *was* a true
|
||||
// dead duplicate and has been deleted (see AUDIT_APPSTORE.md P0-3).
|
||||
// Owns its own AVAudioEngine + AVAudioSession, downsamples to 16 kHz
|
||||
// mono Float32 on the audio thread (same as `AudioCaptureService`), and
|
||||
// feeds `AudioBufferSnapshot` to the shared `ASRService` (the same
|
||||
// pipeline the real keyboard extension
|
||||
// uses, so the preview exercises the *real* iOS speech APIs, not a
|
||||
// stub). Without this the in-app preview was a hardcoded transcript
|
||||
// and "did you actually call SFSpeechRecognizer?" was a fair review
|
||||
// note.
|
||||
//
|
||||
// Why not reuse `AudioCaptureService` from the extension? It lives in
|
||||
// `OSGKeyboardExt`, an `app-extension` target — the main app can't
|
||||
// import its symbols. We could move it to `OSGKeyboardShared`, but
|
||||
// `AVAudioSession` lifecycle differs enough between a keyboard
|
||||
// extension (no background, no recording entitlement surprise) and a
|
||||
// foreground app that a copy here is the lesser evil.
|
||||
|
||||
import Foundation
|
||||
import AVFoundation
|
||||
import Speech
|
||||
import os
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
private enum LiveCaptureGatePhase: Equatable {
|
||||
case idle
|
||||
case recording
|
||||
case draining
|
||||
}
|
||||
|
||||
/// Thread-safe relay so the AVAudioEngine tap can yield snapshots without
|
||||
/// hopping through `@MainActor` (which adds latency and can reorder frames).
|
||||
private final class CaptureStreamRelay: @unchecked Sendable {
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
private var continuation: AsyncStream<AudioBufferSnapshot>.Continuation?
|
||||
|
||||
func bind(_ continuation: AsyncStream<AudioBufferSnapshot>.Continuation) {
|
||||
lock.withLock { self.continuation = continuation }
|
||||
}
|
||||
|
||||
func yield(_ snapshot: AudioBufferSnapshot) {
|
||||
_ = lock.withLock { continuation?.yield(snapshot) }
|
||||
}
|
||||
|
||||
func finish() {
|
||||
lock.withLock {
|
||||
continuation?.finish()
|
||||
continuation = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
public final class LiveDictationController: ObservableObject {
|
||||
|
||||
public enum Phase: Equatable {
|
||||
case idle
|
||||
case requestingPermission
|
||||
case recording
|
||||
case processing
|
||||
case denied(String)
|
||||
case error(String)
|
||||
}
|
||||
|
||||
@Published public private(set) var phase: Phase = .idle
|
||||
/// Normalized 0...1 RMS for the disc level meter. Polled from the
|
||||
/// audio tap via `Task { @MainActor in ... }` — the tap itself
|
||||
/// runs on a real-time audio thread, so we never touch published
|
||||
/// state from there.
|
||||
@Published public private(set) var level: Double = 0
|
||||
@Published public private(set) var currentPartial: String = ""
|
||||
@Published public private(set) var errorMessage: String?
|
||||
|
||||
/// Set when a `.final` ASR event lands. The owning sheet observes
|
||||
/// this and appends the text to its textbox, then clears it so the
|
||||
/// next recording starts from zero.
|
||||
@Published public var lastFinal: String = ""
|
||||
|
||||
private let asr: ASRService
|
||||
private var audioEngine = AVAudioEngine()
|
||||
/// `internal` (not `private`) so the regression test in
|
||||
/// `OSGKeyboardTests/PreviewASRControllerStateTests.swift` can
|
||||
/// install a known consumer task and assert `stop()` doesn't
|
||||
/// cancel it. The class is `@MainActor`-isolated, so the
|
||||
/// natural Swift 6 isolation rules still prevent production
|
||||
/// code outside the class from racing on it.
|
||||
public var asrTask: Task<Void, Never>?
|
||||
private let streamRelay = CaptureStreamRelay()
|
||||
private var chunkedPipeline: ChunkedUtterancePipeline?
|
||||
private let captureGate = OSAllocatedUnfairLock(initialState: LiveCaptureGatePhase.idle)
|
||||
private let drainTracker = FlowCaptureDrainTracker()
|
||||
private var audioConverter: AVAudioConverter?
|
||||
private var targetFormat: AVAudioFormat?
|
||||
private var hwFormat: AVAudioFormat?
|
||||
private var didConfigureAudioSession = false
|
||||
private var didInstallTap = false
|
||||
|
||||
public init(asr: ASRService? = nil) {
|
||||
self.asr = asr ?? ASRServiceFactory.make(store: AppGroupStore())
|
||||
}
|
||||
|
||||
/// Start dictation using a persisted settings locale id (`auto`, `zh-Hans`, …).
|
||||
public func start(localeId: String) async {
|
||||
await start(locale: SpeechLocaleResolver.resolve(localeId))
|
||||
}
|
||||
|
||||
public func start(locale: Locale) async {
|
||||
// Re-entry guard: ignore taps that arrive while we're already
|
||||
// running. (The sheet's `cyclePhase` is also guarded, but
|
||||
// async race windows are easier to lock down here.)
|
||||
switch phase {
|
||||
case .recording, .requestingPermission, .processing:
|
||||
return
|
||||
default:
|
||||
break
|
||||
}
|
||||
// Cancel any leftover consumer task from a previous recording.
|
||||
// Normally `stop()` lets the task run to completion (so it can
|
||||
// see the `.final` and transition out of `.processing`), but if
|
||||
// the user smashed the disc twice — stop, then immediately
|
||||
// start — the previous task might still be draining. Cancel it
|
||||
// here so we don't have two consumer tasks fighting over the
|
||||
// same `events` stream.
|
||||
asrTask?.cancel()
|
||||
asrTask = nil
|
||||
if let pipeline = chunkedPipeline {
|
||||
Task { await pipeline.cancel() }
|
||||
}
|
||||
chunkedPipeline = nil
|
||||
teardownCapturePipeline()
|
||||
phase = .requestingPermission
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
errorMessage = nil
|
||||
level = 0
|
||||
|
||||
// 1. Microphone permission. The helper is `nonisolated` so the
|
||||
// (iOS < 17) callback closure does not inherit `@MainActor` —
|
||||
// `AVAudioSession.requestRecordPermission` delivers on a TCC
|
||||
// reply queue, and a `@MainActor`-inferred closure body there
|
||||
// hits `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift`.
|
||||
let micGranted = await Self.requestMicrophonePermission()
|
||||
guard micGranted else {
|
||||
phase = .denied(NSLocalizedString("keyboard.denied.mic", comment: ""))
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Speech recognition permission. Same reasoning as above:
|
||||
// the callback fires on TCC's reply queue, NOT the main queue.
|
||||
let speechGranted = await Self.requestSpeechRecognitionPermission()
|
||||
guard speechGranted else {
|
||||
phase = .denied(NSLocalizedString("keyboard.denied.speech", comment: ""))
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Audio session — only configure once per process.
|
||||
//
|
||||
// Category is `.record` (not `.playAndRecord`) because the
|
||||
// preview never plays back audio — it just records from the
|
||||
// mic and hands the buffers to `SpeechAnalyzer`. On the
|
||||
// iOS Simulator, `.playAndRecord` requires the
|
||||
// `AURemoteIO` Audio Unit's *output* side to also be
|
||||
// enabled, but the simulator's "speaker" reports a 0 Hz
|
||||
// hardware format, so `AURemoteIO::enable` fails with
|
||||
// `kAudioUnitErr_FormatNotSupported` (-10851) and any
|
||||
// subsequent `installTap` traps with "Failed to create tap
|
||||
// due to format mismatch". `.record` skips the output
|
||||
// side entirely, so the simulator can record.
|
||||
//
|
||||
// The real keyboard extension (`OSGKeyboardExt`) keeps
|
||||
// `.playAndRecord` because it runs on a real device where
|
||||
// the output side has a real hardware format, and may want
|
||||
// to play click sounds / haptic feedback. Only the preview
|
||||
// needs the simulator-friendly category.
|
||||
if !didConfigureAudioSession {
|
||||
do {
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
try session.setCategory(.record,
|
||||
mode: .measurement,
|
||||
options: [])
|
||||
try session.setActive(true, options: .notifyOthersOnDeactivation)
|
||||
didConfigureAudioSession = true
|
||||
} catch {
|
||||
debug("audio session failed: \(error.localizedDescription)")
|
||||
phase = .error(String.localizedStringWithFormat(
|
||||
NSLocalizedString("preview.error.audioSession", comment: ""),
|
||||
error.localizedDescription
|
||||
))
|
||||
return
|
||||
}
|
||||
}
|
||||
// The route may have changed while the preview was closed. A fresh
|
||||
// engine created after session activation avoids a stale input node.
|
||||
audioEngine = AVAudioEngine()
|
||||
|
||||
// 4. Spin up the engine + ASR.
|
||||
phase = .recording
|
||||
startEngineAndASR(locale: locale)
|
||||
}
|
||||
|
||||
public func stop() {
|
||||
// Don't `asrTask?.cancel()` here — see the comment in
|
||||
// `startEngineAndASR` for the full rationale. Short version:
|
||||
// cancelling the consumer task at the same moment we close the
|
||||
// audio stream also triggers the producer's
|
||||
// `continuation.onTermination → self?.cancel()` cascade, which
|
||||
// marks the producer's outer task as cancelled and skips the
|
||||
// `.final` event. The UI is then left in `.processing` forever
|
||||
// because no one schedules the transition out. The consumer
|
||||
// task naturally exits when `events` finishes, so the right
|
||||
// thing is to let it run.
|
||||
//
|
||||
// If a previous `asrTask` is somehow still running (e.g. the
|
||||
// user smashed the disc twice quickly), `start()` cancels it
|
||||
// at the entry point as a safety net.
|
||||
let partial = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !partial.isEmpty && lastFinal.isEmpty {
|
||||
lastFinal = partial
|
||||
currentPartial = ""
|
||||
}
|
||||
if phase == .recording {
|
||||
phase = .processing
|
||||
}
|
||||
|
||||
Task { @MainActor [weak self] in
|
||||
await self?.drainTailAndTeardownCapture()
|
||||
}
|
||||
|
||||
// Safety net: if the ASR pipeline never produces a `.final`
|
||||
// (analyzer hang, system glitch, dropped continuation), force
|
||||
// the UI back to idle after a short delay so the user isn't
|
||||
// stuck. Normal recordings complete well under a second, so
|
||||
// the 3-second budget is only hit on the unhappy path; if the
|
||||
// pipeline finishes first and flips the phase to `.idle` (or
|
||||
// `.error`), the check below no-ops.
|
||||
Task { @MainActor [weak self] in
|
||||
try? await Task.sleep(for: .seconds(3))
|
||||
guard let self else { return }
|
||||
if self.phase == .processing {
|
||||
let stalePartial = self.currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !stalePartial.isEmpty, self.lastFinal.isEmpty {
|
||||
self.debug("processing timeout, using partial")
|
||||
self.lastFinal = stalePartial
|
||||
self.currentPartial = ""
|
||||
}
|
||||
self.phase = .idle
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public func reset() {
|
||||
// Called by the sheet after appending `lastFinal` to the textbox,
|
||||
// so the next recording can produce a fresh final without us
|
||||
// double-appending.
|
||||
lastFinal = ""
|
||||
if phase == .processing {
|
||||
phase = .idle
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Engine + ASR
|
||||
|
||||
private func startEngineAndASR(locale: Locale) {
|
||||
let inputNode = audioEngine.inputNode
|
||||
let hwFormat = inputNode.inputFormat(forBus: 0)
|
||||
|
||||
// Pre-flight check: a placeholder / unconfigured input bus
|
||||
// reports `sampleRate == 0` (or `channelCount == 0`).
|
||||
// `installTap` on such a bus traps with "Failed to create
|
||||
// tap due to format mismatch" (an NSException, not a Swift
|
||||
// `Error`, so we can't `try`/`catch` it). The safest fix
|
||||
// is to refuse the tap up front and surface a clear
|
||||
// `.error` phase instead of crashing the app. We've seen
|
||||
// this on the iOS Simulator when the host's microphone
|
||||
// permission isn't granted to CoreSimulator, and on
|
||||
// devices where the audio session is in an unexpected
|
||||
// state from a previous foreground/background transition.
|
||||
guard hwFormat.sampleRate > 0, hwFormat.channelCount > 0 else {
|
||||
debug("invalid hardware format sr=\(hwFormat.sampleRate) ch=\(hwFormat.channelCount)")
|
||||
phase = .error(
|
||||
String.localizedStringWithFormat(
|
||||
NSLocalizedString("preview.error.micUnavailable", comment: ""),
|
||||
hwFormat.sampleRate,
|
||||
Int(hwFormat.channelCount)
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
let targetSampleRate: Double = 16_000
|
||||
guard let targetFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32,
|
||||
sampleRate: targetSampleRate,
|
||||
channels: 1,
|
||||
interleaved: false
|
||||
) else {
|
||||
phase = .error(NSLocalizedString("preview.error.formatCreate", comment: ""))
|
||||
return
|
||||
}
|
||||
guard let converter = AVAudioConverter(from: hwFormat, to: targetFormat) else {
|
||||
debug("converter creation failed")
|
||||
phase = .error(NSLocalizedString("preview.error.converterCreate", comment: ""))
|
||||
return
|
||||
}
|
||||
|
||||
audioConverter = converter
|
||||
self.targetFormat = targetFormat
|
||||
self.hwFormat = hwFormat
|
||||
drainTracker.reset()
|
||||
captureGate.withLock { $0 = .recording }
|
||||
|
||||
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
|
||||
streamRelay.bind(continuation)
|
||||
|
||||
// Tap the hardware input. The closure passed to `installTap` runs
|
||||
// on the AVAudioEngine real-time audio thread. In Swift 6 strict
|
||||
// concurrency, a closure literal defined inside a `@MainActor`
|
||||
// method inherits `@MainActor` isolation, which would trip
|
||||
// `dispatch_assert_queue_fail` on first invocation from the
|
||||
// audio thread. The fix is to build the actual tap body in a
|
||||
// `nonisolated` helper (`makeAudioTapBlock`) and have the
|
||||
// installTap closure be a single function reference — function
|
||||
// references never carry inferred isolation, so the dispatch
|
||||
// runtime is happy and the body runs wherever AVAudioEngine
|
||||
// wants it (the audio thread).
|
||||
let onMeter: @Sendable (Double) -> Void = { [weak self] meter in
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
// Lightweight smoothing so the disc ring doesn't jitter.
|
||||
self.level = self.level * 0.55 + meter * 0.45
|
||||
}
|
||||
}
|
||||
let relay = streamRelay
|
||||
let gate = captureGate
|
||||
let tracker = drainTracker
|
||||
let policy = FlowCaptureTailDrainPolicy.flowDefault
|
||||
let onSnapshot: @Sendable (AudioBufferSnapshot) -> Void = { snapshot in
|
||||
let phase = gate.withLock { $0 }
|
||||
guard phase == .recording || phase == .draining else { return }
|
||||
relay.yield(snapshot)
|
||||
if phase == .draining {
|
||||
tracker.noteAudio(samples: snapshot.samples, policy: policy)
|
||||
}
|
||||
}
|
||||
let tap = Self.makeAudioTapBlock(
|
||||
converter: converter,
|
||||
targetFormat: targetFormat,
|
||||
hwFormat: hwFormat,
|
||||
onMeter: onMeter,
|
||||
onSnapshot: onSnapshot
|
||||
)
|
||||
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
|
||||
didInstallTap = true
|
||||
|
||||
audioEngine.prepare()
|
||||
do {
|
||||
try audioEngine.start()
|
||||
} catch {
|
||||
debug("audio engine start failed: \(error.localizedDescription)")
|
||||
phase = .error(String.localizedStringWithFormat(
|
||||
NSLocalizedString("preview.error.engineStart", comment: ""),
|
||||
error.localizedDescription
|
||||
))
|
||||
return
|
||||
}
|
||||
|
||||
// 5. Pipelined ASR (same chunk path as Flow host).
|
||||
let pipeline = ChunkedUtterancePipeline(asr: asr, locale: locale)
|
||||
chunkedPipeline = pipeline
|
||||
asrTask = Task.detached(priority: .userInitiated) { [weak controller = self] in
|
||||
let outcome = await pipeline.transcribe(stream: stream) { partial in
|
||||
Task { @MainActor in
|
||||
controller?.currentPartial = partial
|
||||
}
|
||||
}
|
||||
// Re-bind `controller` inside the `@MainActor` block so the
|
||||
// weak reference is captured under the right isolation. Swift
|
||||
// 6 strict concurrency otherwise complains about a
|
||||
// task-isolated reference escaping into a main-actor closure.
|
||||
await MainActor.run { [weak controller] in
|
||||
guard let controller else { return }
|
||||
switch outcome {
|
||||
case .success(let success):
|
||||
let trimmed = success.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !trimmed.isEmpty {
|
||||
controller.lastFinal = trimmed
|
||||
controller.currentPartial = ""
|
||||
}
|
||||
if controller.phase == .processing || controller.phase == .recording {
|
||||
controller.phase = .idle
|
||||
}
|
||||
case .failure(let message):
|
||||
controller.debug("asr error: \(message)")
|
||||
controller.teardownCapturePipeline()
|
||||
controller.errorMessage = message
|
||||
controller.phase = .error(message)
|
||||
case .cancelled:
|
||||
if controller.phase == .processing {
|
||||
controller.phase = .idle
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Permission helpers (nonisolated)
|
||||
//
|
||||
// `SFSpeechRecognizer.requestAuthorization` delivers its callback
|
||||
// on a TCC reply queue, NOT the main queue. If we wrap that
|
||||
// callback inline in `start(locale:)` — which is `@MainActor` —
|
||||
// Swift 6 strict concurrency infers the closure body as
|
||||
// `@MainActor`, and the runtime crashes on
|
||||
// `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift` as
|
||||
// soon as TCC calls us back.
|
||||
//
|
||||
// The first attempt (commit `e8a0310`) extracted the entire
|
||||
// permission request into a `nonisolated static func` helper.
|
||||
// That worked in isolation, but the Swift 6 optimizer
|
||||
// inlined those helpers back into `start(locale:)`. After
|
||||
// inlining, the `withCheckedContinuation` body and the
|
||||
// `requestAuthorization` callback were re-typed in the
|
||||
// `@MainActor` context of the caller, and the runtime
|
||||
// assertion came right back — same crash, different symbol:
|
||||
// `closure #1 in closure #2 in PreviewASRController.start(locale:)`.
|
||||
//
|
||||
// The fix that survives inlining is the *function-reference*
|
||||
// pattern, the same one used for `installTap` in
|
||||
// `makeAudioTapBlock` below. The callback is built in a
|
||||
// `nonisolated` static helper that takes a `CheckedContinuation`
|
||||
// and returns the `(Status) -> Void` handler. The body of that
|
||||
// helper has no enclosing actor, so the closure is created in
|
||||
// nonisolated context. When TCC calls us back, the runtime
|
||||
// sees a nonisolated closure on a non-main queue and is happy.
|
||||
//
|
||||
// `cont.resume(...)` is itself thread-safe on
|
||||
// `CheckedContinuation`, so we don't need to hop back to the
|
||||
// main actor before resuming.
|
||||
|
||||
private nonisolated static func requestMicrophonePermission() async -> Bool {
|
||||
// iOS 17+ API; the iOS < 17 fallback (`AVAudioSession.recordPermission`
|
||||
// + `requestRecordPermission` callback) is gone now that the
|
||||
// deployment target is iOS 26.
|
||||
switch AVAudioApplication.shared.recordPermission {
|
||||
case .granted: return true
|
||||
case .denied: return false
|
||||
case .undetermined: return await AVAudioApplication.requestRecordPermission()
|
||||
@unknown default: return false
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func requestSpeechRecognitionPermission() async -> Bool {
|
||||
await withCheckedContinuation { (cont: CheckedContinuation<Bool, Never>) in
|
||||
SFSpeechRecognizer.requestAuthorization(
|
||||
Self.makeSpeechAuthHandler(continuation: cont)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func makeSpeechAuthHandler(
|
||||
continuation: CheckedContinuation<Bool, Never>
|
||||
) -> @Sendable (SFSpeechRecognizerAuthorizationStatus) -> Void {
|
||||
return { status in
|
||||
continuation.resume(returning: status == .authorized)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Audio tap (nonisolated, runs on AVAudioEngine render thread)
|
||||
//
|
||||
// `AVAudioNode.installTap`'s callback fires on the audio engine's
|
||||
// real-time render thread. In Swift 6 strict concurrency, a closure
|
||||
// literal defined inside a `@MainActor` method inherits `@MainActor`
|
||||
// isolation — and `dispatch_assert_queue_fail` fires the moment
|
||||
// the runtime tries to dispatch that closure on a non-main queue.
|
||||
//
|
||||
// The trick is to build the actual tap body in a `nonisolated`
|
||||
// function and have the installTap closure be a *function reference*
|
||||
// to that helper. Function references never carry inferred
|
||||
// isolation, so the dispatch runtime is satisfied and the body
|
||||
// runs wherever AVAudioEngine wants. State updates to
|
||||
// `self.level` and the AsyncStream continuation hop back to the
|
||||
// main actor via `Task { @MainActor in … }`, which is itself
|
||||
// safe to call from a non-isolated context.
|
||||
private nonisolated static func makeAudioTapBlock(
|
||||
converter: AVAudioConverter,
|
||||
targetFormat: AVAudioFormat,
|
||||
hwFormat: AVAudioFormat,
|
||||
onMeter: @Sendable @escaping (Double) -> Void,
|
||||
onSnapshot: @Sendable @escaping (AudioBufferSnapshot) -> Void
|
||||
) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
|
||||
// `@Sendable` on the returned closure makes the Sendable
|
||||
// conformance explicit. `AVAudioNodeTapBlock` is declared as
|
||||
// a plain escaping closure in the SDK; we cast at the call
|
||||
// site via `as @Sendable`.
|
||||
return { buffer, _ in
|
||||
// 1) Level meter from raw hardware buffer.
|
||||
let n = Int(buffer.frameLength)
|
||||
var sumSquares: Float = 0
|
||||
if let channelData = buffer.floatChannelData?[0], n > 0 {
|
||||
for i in 0..<n {
|
||||
let v = channelData[i]
|
||||
sumSquares += v * v
|
||||
}
|
||||
}
|
||||
let rms = n > 0 ? sqrtf(sumSquares / Float(n)) : 0
|
||||
let meter = min(Double(rms) * 4.0, 1.0)
|
||||
onMeter(meter)
|
||||
|
||||
// 2) Downsample to 16 kHz mono Float32 for ASR (matches
|
||||
// `AudioCaptureService` and Apple's `considering:` hint).
|
||||
let outFrames = AVAudioFrameCount(
|
||||
Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate
|
||||
)
|
||||
guard outFrames > 0,
|
||||
let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outFrames)
|
||||
else { return }
|
||||
|
||||
var error: NSError?
|
||||
let status = converter.convert(to: outBuffer, error: &error) { _, outStatus in
|
||||
outStatus.pointee = .haveData
|
||||
return buffer
|
||||
}
|
||||
guard status == .haveData, error == nil, outBuffer.frameLength > 0 else { return }
|
||||
|
||||
let snapshot = AudioBufferSnapshot(buffer: outBuffer)
|
||||
guard !snapshot.samples.isEmpty else { return }
|
||||
onSnapshot(snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
private func drainTailAndTeardownCapture() async {
|
||||
let beganDrain = captureGate.withLock { phase -> Bool in
|
||||
switch phase {
|
||||
case .recording:
|
||||
phase = .draining
|
||||
return true
|
||||
case .draining, .idle:
|
||||
return false
|
||||
}
|
||||
}
|
||||
guard beganDrain else { return }
|
||||
|
||||
drainTracker.beginDrain()
|
||||
|
||||
let policy = FlowCaptureTailDrainPolicy.flowDefault
|
||||
_ = await FlowUtteranceEndCoordinator.awaitTailCapture(
|
||||
tracker: drainTracker,
|
||||
policy: policy
|
||||
)
|
||||
|
||||
// Trailing speech is preserved by the live `.draining` forwarding
|
||||
// loop above. We deliberately do NOT signal `.endOfStream` to the
|
||||
// converter to squeeze its internal filter tail: that both races the
|
||||
// still-running audio-thread tap on the same non-thread-safe converter
|
||||
// and (in reused-converter paths) permanently locks it. The dropped
|
||||
// tail is sub-millisecond and inaudible.
|
||||
streamRelay.finish()
|
||||
teardownCaptureEngine()
|
||||
captureGate.withLock { $0 = .idle }
|
||||
drainTracker.reset()
|
||||
audioConverter = nil
|
||||
targetFormat = nil
|
||||
hwFormat = nil
|
||||
|
||||
try? AVAudioSession.sharedInstance().setActive(
|
||||
false,
|
||||
options: .notifyOthersOnDeactivation
|
||||
)
|
||||
}
|
||||
|
||||
private func teardownCaptureEngine() {
|
||||
if didInstallTap {
|
||||
audioEngine.inputNode.removeTap(onBus: 0)
|
||||
didInstallTap = false
|
||||
}
|
||||
if audioEngine.isRunning {
|
||||
audioEngine.stop()
|
||||
}
|
||||
}
|
||||
|
||||
private func teardownCapturePipeline() {
|
||||
teardownCaptureEngine()
|
||||
streamRelay.finish()
|
||||
}
|
||||
|
||||
private func debug(_ message: String) {
|
||||
#if DEBUG
|
||||
print("🎙️[LiveDictationController] \(message)")
|
||||
#endif
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
// TipPurchaseManager.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Optional ¥30 consumable tip via StoreKit 2. Voluntary support only —
|
||||
// no feature gates, no App Group sync, no restore (Apple consumable rules).
|
||||
|
||||
import Foundation
|
||||
import Combine
|
||||
import StoreKit
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public enum TipPurchaseState: Equatable {
|
||||
case idle
|
||||
case loading
|
||||
case purchasing
|
||||
case succeeded
|
||||
case failed(String)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
public final class TipPurchaseManager: ObservableObject {
|
||||
public static let shared = TipPurchaseManager()
|
||||
|
||||
@Published public private(set) var product: Product?
|
||||
@Published public private(set) var purchaseState: TipPurchaseState = .idle
|
||||
@Published public private(set) var supportCount: Int
|
||||
|
||||
private var transactionUpdatesTask: Task<Void, Never>?
|
||||
|
||||
public init(defaults: UserDefaults = .standard) {
|
||||
supportCount = defaults.integer(forKey: TipProduct.supportCountDefaultsKey)
|
||||
transactionUpdatesTask = Task { [weak self] in
|
||||
await self?.listenForTransactionUpdates()
|
||||
}
|
||||
Task { await loadProducts() }
|
||||
}
|
||||
|
||||
deinit {
|
||||
transactionUpdatesTask?.cancel()
|
||||
}
|
||||
|
||||
/// Loads the tip product from the App Store / StoreKit Test configuration.
|
||||
public func loadProducts() async {
|
||||
guard purchaseState != .purchasing else { return }
|
||||
purchaseState = .loading
|
||||
do {
|
||||
let products = try await Product.products(for: TipProduct.allProductIDs)
|
||||
product = products.first(where: { $0.id == TipProduct.supportID })
|
||||
purchaseState = product == nil
|
||||
? .failed(SharedL10n.string("tip.error.productUnavailable"))
|
||||
: .idle
|
||||
} catch {
|
||||
product = nil
|
||||
purchaseState = .failed(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts the StoreKit purchase sheet for the support tip.
|
||||
public func purchase() async {
|
||||
guard let product else {
|
||||
purchaseState = .failed(SharedL10n.string("tip.error.productUnavailable"))
|
||||
return
|
||||
}
|
||||
purchaseState = .purchasing
|
||||
do {
|
||||
let result = try await product.purchase()
|
||||
switch result {
|
||||
case .success(let verification):
|
||||
let transaction = try Self.verified(verification)
|
||||
await handleCompletedTip(transaction)
|
||||
purchaseState = .succeeded
|
||||
case .userCancelled:
|
||||
purchaseState = .idle
|
||||
case .pending:
|
||||
purchaseState = .failed(SharedL10n.string("tip.error.pending"))
|
||||
@unknown default:
|
||||
purchaseState = .failed(SharedL10n.string("tip.error.unknown"))
|
||||
}
|
||||
} catch {
|
||||
purchaseState = .failed(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
/// Clears transient success / failure state after the UI acknowledges it.
|
||||
public func acknowledgePurchaseState() {
|
||||
switch purchaseState {
|
||||
case .succeeded, .failed:
|
||||
purchaseState = .idle
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
private func listenForTransactionUpdates() async {
|
||||
for await update in Transaction.updates {
|
||||
guard let transaction = try? Self.verified(update),
|
||||
transaction.productID == TipProduct.supportID else { continue }
|
||||
await handleCompletedTip(transaction)
|
||||
}
|
||||
}
|
||||
|
||||
private func handleCompletedTip(_ transaction: Transaction) async {
|
||||
await transaction.finish()
|
||||
recordSupportPurchase()
|
||||
}
|
||||
|
||||
private func recordSupportPurchase() {
|
||||
supportCount += 1
|
||||
UserDefaults.standard.set(supportCount, forKey: TipProduct.supportCountDefaultsKey)
|
||||
}
|
||||
|
||||
private static func verified<T>(_ result: VerificationResult<T>) throws -> T {
|
||||
switch result {
|
||||
case .verified(let value):
|
||||
return value
|
||||
case .unverified:
|
||||
throw TipPurchaseError.failedVerification
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private enum TipPurchaseError: Error {
|
||||
case failedVerification
|
||||
}
|
||||
Reference in New Issue
Block a user