feat: migrate on-device Qwen3 ASR to CoreML for background Flow dictation

Replace MLX GPU inference with CoreML bundles so transcription continues
while the host app is backgrounded. Adds model download and warm-up,
vendored Qwen3Speech, and updates onboarding, settings, and copy for the
~1.6 GB CoreML package (iOS 18+).
This commit is contained in:
Rocky
2026-06-23 00:46:58 +08:00
parent 5e5122f172
commit df1c5ff32c
160 changed files with 22080 additions and 492 deletions
@@ -0,0 +1,41 @@
// SharedL10n.swift
// OSGKeyboard · Shared
//
// Localized strings shipped inside the shared framework (Shared.strings).
// Respects the in-app UI language override from App Group settings.
import Foundation
public enum SharedL10n {
private static let table = "Shared"
private static let container = Bundle(for: SharedBundleToken.self)
public static func string(
_ key: String,
language: AppUILanguage? = nil
) -> String {
let lang = language ?? AppGroupStore().uiLanguage
let bundle = AppUILanguage.localizedBundle(in: container, language: lang)
return NSLocalizedString(
key,
tableName: table,
bundle: bundle,
value: key,
comment: ""
)
}
public static func format(
_ key: String,
language: AppUILanguage? = nil,
_ args: CVarArg...
) -> String {
String(
format: string(key, language: language),
locale: Locale.current,
arguments: args
)
}
}
private final class SharedBundleToken {}
@@ -0,0 +1,87 @@
// AppUILanguage.swift
// OSGKeyboard · Shared
//
// In-app UI language override (main app + keyboard extension strings).
// Distinct from `localeId`, which controls speech recognition language.
import Foundation
public enum AppUILanguage: String, CaseIterable, Identifiable, Sendable, Codable {
case auto
case english = "en"
case chinese = "zh-Hans"
public var id: String { rawValue }
public var labelKey: String {
switch self {
case .auto: return "settings.appLanguage.auto"
case .english: return "settings.appLanguage.english"
case .chinese: return "settings.appLanguage.chinese"
}
}
/// Locale for SwiftUI `.environment(\.locale, )` in the host app.
public var swiftUILocale: Locale {
switch self {
case .auto:
return Locale.autoupdatingCurrent
case .english:
return Locale(identifier: "en")
case .chinese:
return Locale(identifier: "zh-Hans")
}
}
/// `.lproj` folder name used for manual bundle lookups (extension).
public func resolvedLanguageCode(
preferredLanguages: [String] = Locale.preferredLanguages
) -> String {
switch self {
case .english:
return "en"
case .chinese:
return "zh-Hans"
case .auto:
if preferredLanguages.contains(where: { $0.hasPrefix("zh") }) {
return "zh-Hans"
}
return "en"
}
}
public static func fromStored(_ raw: String?) -> AppUILanguage {
guard let raw, let value = AppUILanguage(rawValue: raw) else { return .auto }
return value
}
/// Picks the best-matching `.lproj` inside `container` for this preference.
public static func localizedBundle(
in container: Bundle,
language: AppUILanguage,
preferredLanguages: [String] = Locale.preferredLanguages
) -> Bundle {
let code = language.resolvedLanguageCode(preferredLanguages: preferredLanguages)
guard let path = container.path(forResource: code, ofType: "lproj"),
let bundle = Bundle(path: path) else {
return container
}
return bundle
}
public static func localizedString(
_ key: String,
tableName: String?,
bundle container: Bundle,
language: AppUILanguage = AppGroupStore().uiLanguage
) -> String {
let bundle = localizedBundle(in: container, language: language)
return NSLocalizedString(
key,
tableName: tableName,
bundle: bundle,
value: key,
comment: ""
)
}
}
@@ -9,18 +9,37 @@ public enum EngineServiceLabel {
public static func summary(
engineMode: String,
providerId: String,
model: String
model: String,
localASRBackend: LocalASRBackend = .speechAnalyzer,
language: AppUILanguage? = nil
) -> String {
let isChinese = Locale.preferredLanguages.first?.hasPrefix("zh") == true
let prefix = isChinese ? "当前:" : "Active: "
let lang = language ?? AppGroupStore().uiLanguage
if engineMode == "local" {
return isChinese
? "\(prefix)本地引擎 · Apple SpeechAnalyzer"
: "\(prefix)On-device · Apple SpeechAnalyzer"
let asrName = asrDisplayName(for: localASRBackend, language: lang)
return SharedL10n.format("engine.summary.local", language: lang, asrName)
}
let providerName = ProviderDisplayName.name(for: providerId)
let providerName = ProviderDisplayName.name(for: providerId, language: lang)
let trimmedModel = model.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmedModel.isEmpty { return "\(prefix)\(providerName)" }
return "\(prefix)\(providerName) · \(trimmedModel)"
if trimmedModel.isEmpty {
return SharedL10n.format("engine.summary.cloud", language: lang, providerName)
}
return SharedL10n.format(
"engine.summary.cloudWithModel",
language: lang,
providerName,
trimmedModel
)
}
private static func asrDisplayName(
for backend: LocalASRBackend,
language: AppUILanguage
) -> String {
switch backend {
case .speechAnalyzer:
return SharedL10n.string("engine.asr.appleSpeech", language: language)
case .qwen3ASR:
return SharedL10n.string("model.qwen3asr.name", language: language)
}
}
}
@@ -0,0 +1,69 @@
// FlowUtteranceChunkConfig.swift
// OSGKeyboard · Shared
//
// Chunking policy for pipelined Flow utterance ASR (up to 3 minutes).
import Foundation
public struct FlowUtteranceChunkConfig: Sendable, Equatable {
/// Target maximum duration per ASR chunk.
public let maxChunkDurationSeconds: TimeInterval
/// Tail overlap fed into the next chunk for boundary dedup when stitching.
public let overlapDurationSeconds: TimeInterval
/// After hitting the max window, wait up to this long for a pause before hard-splitting.
public let pauseExtensionMaxSeconds: TimeInterval
/// RMS below this is treated as a pause candidate (Float32 mono @ 16 kHz).
public let pauseRMSThreshold: Float
public let sampleRate: Int
public init(
maxChunkDurationSeconds: TimeInterval,
overlapDurationSeconds: TimeInterval,
pauseExtensionMaxSeconds: TimeInterval,
pauseRMSThreshold: Float,
sampleRate: Int
) {
self.maxChunkDurationSeconds = maxChunkDurationSeconds
self.overlapDurationSeconds = overlapDurationSeconds
self.pauseExtensionMaxSeconds = pauseExtensionMaxSeconds
self.pauseRMSThreshold = pauseRMSThreshold
self.sampleRate = sampleRate
}
public var maxChunkSamples: Int {
Int(maxChunkDurationSeconds * Double(sampleRate))
}
public var overlapSamples: Int {
Int(overlapDurationSeconds * Double(sampleRate))
}
public var pauseExtensionSamples: Int {
Int(pauseExtensionMaxSeconds * Double(sampleRate))
}
/// Default for keyboard Flow utterances ( 3 min, pipelined ASR).
public static let flowDefault = FlowUtteranceChunkConfig(
maxChunkDurationSeconds: 30,
overlapDurationSeconds: 0.5,
pauseExtensionMaxSeconds: 2,
pauseRMSThreshold: 0.015,
sampleRate: 16_000
)
}
public struct UtteranceAudioChunk: Sendable, Equatable {
public let index: Int
public let samples: [Float]
public let isLast: Bool
public init(index: Int, samples: [Float], isLast: Bool) {
self.index = index
self.samples = samples
self.isLast = isLast
}
public var durationSeconds: Double {
Double(samples.count) / 16_000.0
}
}
@@ -0,0 +1,56 @@
// LocalASRBackend.swift
// OSGKeyboard · Shared
//
// Identifies which on-device speech recognition engine to use when the
// user picks the "local" engine (no cloud LLM polish). The shared
// factory `ASRServiceFactory` dispatches on this enum; the settings UI
// renders it as a picker.
//
// Why an enum in `Shared` rather than living next to the concrete
// `ASRService` implementations: the value must be serialisable into
// the App Group store (so the keyboard extension can observe the
// selection), exposed via `ProviderConfig` (UI binding) and consumed
// by every layer that asks for an ASR backend.
import Foundation
public enum LocalASRBackend: String, CaseIterable, Identifiable, Sendable, Codable {
/// iOS 26 `SpeechAnalyzer` + `DictationTranscriber`. Always
/// on-device, no asset download, ships with iOS. Default for every
/// fresh install anything else is opt-in.
case speechAnalyzer
/// Qwen3-ASR-0.6B via CoreML (Neural Engine + CPU). Stronger on Chinese
/// dialects and noisy audio than `SpeechAnalyzer`, works in Flow while
/// the host app is backgrounded, but requires a ~1.6 GB download on first
/// use and iOS 18+.
case qwen3ASR
public var id: String { rawValue }
/// Localisation key for the human label in the settings picker.
public var labelKey: String {
switch self {
case .speechAnalyzer: return "asr.backend.speechAnalyzer.label"
case .qwen3ASR: return "asr.backend.qwen3.label"
}
}
/// Localisation key for the one-line subtitle shown under the label.
public var blurbKey: String {
switch self {
case .speechAnalyzer: return "asr.backend.speechAnalyzer.blurb"
case .qwen3ASR: return "asr.backend.qwen3.blurb"
}
}
/// Whether this backend needs the user to download a model file
/// before it can run. Used to gate the "Downloading Qwen3-ASR" UI
/// in a follow-up; for now we just expose the flag.
public var requiresModelDownload: Bool {
switch self {
case .speechAnalyzer: return false
case .qwen3ASR: return true
}
}
}
@@ -0,0 +1,54 @@
// OnDeviceModel.swift
// OSGKeyboard · Shared
//
// Identity of an on-device model the host app downloads and the
// keyboard extension observes via App Group flags (the extension
// cannot read the main app's Caches directory).
import Foundation
public enum OnDeviceModel: String, CaseIterable, Identifiable, Sendable {
case qwen3ASR
public var id: String { rawValue }
/// CoreML inference bundle (`aufklarer/Qwen3-ASR-CoreML`), derived from
/// official `Qwen/Qwen3-ASR-0.6B`.
public var repoId: String {
switch self {
case .qwen3ASR: return "aufklarer/Qwen3-ASR-CoreML"
}
}
/// Tokenizer files (vocab / merges) pulled from the upstream Qwen repo.
public var tokenizerRepoId: String {
switch self {
case .qwen3ASR: return "Qwen/Qwen3-ASR-0.6B"
}
}
public var displayName: String {
switch self {
case .qwen3ASR: return "Qwen3-ASR 0.6B (CoreML)"
}
}
public var approximateSizeMB: Int {
switch self {
case .qwen3ASR: return 1_600
}
}
public var compactSizeLabel: String {
"\(approximateSizeMB)M"
}
/// Settings list title: model name plus compact size.
public var listTitle: String {
"\(displayName) · \(compactSizeLabel)"
}
public var repoAndSizeLabel: String {
"\(repoId) · \(approximateSizeMB) MB"
}
}
+31 -3
View File
@@ -30,6 +30,11 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
static let hasCompletedOnboarding = "config.hasCompletedOnboarding"
static let onboardingPage = "config.onboardingPage"
static let hasAcknowledgedCloudSharing = "config.hasAcknowledgedCloudSharing"
// Which on-device ASR engine to use when `engineMode == "local"`.
// Persisted in the App Group so the keyboard can read the
// selection even though it never instantiates the backend itself.
static let localASRBackend = "config.localASRBackend"
static let uiLanguage = "config.uiLanguage"
}
@Published public var providerId: String {
@@ -64,8 +69,8 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
@Published public var localeId: String {
didSet { defaults.set(localeId, forKey: Key.localeId) }
}
/// "local" on-device ASR only, no LLM polishing.
/// "cloud" ASR + LLM polish (default).
/// "local" on-device ASR only (raw transcript delivery).
/// "cloud" ASR + LLM polish (always on; modeId kept for compatibility).
@Published public var engineMode: String {
didSet { defaults.set(engineMode, forKey: Key.engineMode) }
}
@@ -85,6 +90,16 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
@Published public var hasAcknowledgedCloudSharing: Bool {
didSet { defaults.set(hasAcknowledgedCloudSharing, forKey: Key.hasAcknowledgedCloudSharing) }
}
/// Which on-device ASR engine backs the "local" engine mode. Only
/// consulted when `isLocalEngine == true`; the cloud engine always
/// uses `SpeechAnalyzer`.
@Published public var localASRBackend: LocalASRBackend {
didSet { defaults.set(localASRBackend.rawValue, forKey: Key.localASRBackend) }
}
/// Host-app UI language. Also mirrored to the App Group for the keyboard extension.
@Published public var uiLanguage: AppUILanguage {
didSet { defaults.set(uiLanguage.rawValue, forKey: Key.uiLanguage) }
}
public var isConfigured: Bool {
// Local engine (on-device ASR only) doesn't need an API key,
@@ -96,7 +111,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
return !baseURL.isEmpty && !apiKey.isEmpty && !model.isEmpty
}
/// On-device ASR only no cloud LLM polish.
/// On-device ASR only; no cloud API required.
public var isLocalEngine: Bool { engineMode == "local" }
/// The system prompt the user *sees* in the editor fall back to the
@@ -131,6 +146,19 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
let savedPage = resolvedDefaults.integer(forKey: Key.onboardingPage)
self.onboardingPage = savedPage > 0 ? savedPage : 0
self.hasAcknowledgedCloudSharing = resolvedDefaults.bool(forKey: Key.hasAcknowledgedCloudSharing)
// Tolerate missing / unknown raw values (e.g. an enum case that
// was renamed in a later build) by falling back to the default
// rather than crashing inside `RawRepresentable.init`.
let rawBackend = resolvedDefaults.string(forKey: Key.localASRBackend) ?? LocalASRBackend.speechAnalyzer.rawValue
self.localASRBackend = LocalASRBackend(rawValue: rawBackend) ?? .speechAnalyzer
self.uiLanguage = AppUILanguage.fromStored(
resolvedDefaults.string(forKey: Key.uiLanguage)
)
// Cloud no longer exposes off/transcribe; migrate legacy values.
if self.engineMode == "cloud", self.modeId != "polish" {
self.modeId = "polish"
}
}
/// Read the API key from the Keychain, falling back to a one-time
@@ -0,0 +1,17 @@
// TranscriptionDelivery.swift
// OSGKeyboard · Shared
//
// Host-app keyboard handoff payload: final text plus an optional soft
// warning when cloud polish failed but the raw transcript is still delivered.
import Foundation
public struct TranscriptionDelivery: Sendable, Equatable {
public let text: String
public let polishWarning: String?
public init(text: String, polishWarning: String? = nil) {
self.text = text
self.polishWarning = polishWarning
}
}
+240 -13
View File
@@ -16,6 +16,7 @@
import Foundation
import AVFoundation
import CoreMedia
import Speech
import os
@@ -42,6 +43,63 @@ public protocol ASRService: Sendable {
/// Cancel any in-flight recognition and tear down its tasks.
func cancel()
/// Clears cancellation / cached session state before a new utterance.
func resetForNewUtterance()
/// Transcribe one PCM chunk (Flow pipelined path). Default wraps `transcribe(stream:)`.
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult
}
public enum ASRChunkResult: Sendable, Equatable {
case success(String)
case failure(String)
case cancelled
}
extension ASRService {
public func resetForNewUtterance() {}
public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
guard !samples.isEmpty else { return .success("") }
if Task.isCancelled { return .cancelled }
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
continuation.yield(snapshot)
continuation.finish()
var lastPartial = ""
var finalText = ""
var failure: String?
for await event in transcribe(stream: stream, locale: locale) {
if Task.isCancelled { return .cancelled }
switch event {
case .capability:
break
case .partial(let text):
lastPartial = text
case .final(let text):
finalText = text
case .error(let message):
failure = message
}
}
if let failure {
return .failure(failure)
}
let trimmed = finalText.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty {
return .success(trimmed)
}
let partial = lastPartial.trimmingCharacters(in: .whitespacesAndNewlines)
if !partial.isEmpty {
return .success(partial)
}
return .success("")
}
}
public enum ASREvent: Sendable, Equatable {
@@ -58,13 +116,68 @@ public enum ASREvent: Sendable, Equatable {
// MARK: - Factory
public enum ASRServiceFactory {
/// Returns the ASR backend. With iOS 26 as the deployment target,
/// there is exactly one backend (`SpeechAnalyzer`).
/// Registry of backend-specific providers. The host app installs
/// a provider for the Qwen3-ASR backend at launch time (the
/// `Qwen3ASRProvider` lives in the app target because linking
/// `Qwen3ASR` pulls in mlx-swift, which the shared framework
/// deliberately stays off to keep `APPLICATION_EXTENSION_API_ONLY`
/// clean). The shared framework always provides a built-in
/// `SpeechAnalyzer` provider; custom providers override it.
///
/// `nonisolated(unsafe)` because the only writer is
/// `OSGKeyboardApp.init` (single-threaded, runs once at launch).
/// After launch, all callers read the dictionary from any
/// actor.
public nonisolated(unsafe) static var providers: [LocalASRBackend: any ASRServiceProvider] = [
.speechAnalyzer: SpeechAnalyzerProvider()
]
/// Returns the ASR backend chosen by the user. The cloud engine
/// always uses the iOS `SpeechAnalyzer` path it has the lowest
/// latency and never hits the network, which matches the user's
/// expectation that "ASR" is the local half of the pipeline
/// regardless of where the LLM polish happens.
///
/// For the local engine, we honour `LocalASRBackend`:
/// - `.speechAnalyzer` (default) on-device iOS pipeline.
/// - `.qwen3ASR` CoreML-backed Qwen3-ASR via `soniqo/speech-swift` (host app only)
/// (registered by the host app at launch).
public static func make(
engineMode: String,
localBackend: LocalASRBackend = .speechAnalyzer
) -> ASRService {
if engineMode == "local" {
if let provider = providers[localBackend] {
return provider.make()
}
}
return SpeechAnalyzerASR()
}
/// Back-compat overload for callers that only ever want the
/// SpeechAnalyzer path. The previous single-backend build used
/// this signature; new code should pass the engine mode explicitly
/// so the user's selection is honoured.
public static func make() -> ASRService {
SpeechAnalyzerASR()
}
}
/// Backend-specific ASR factory. The shared framework ships a default
/// `SpeechAnalyzerProvider`; the host app installs a `Qwen3ASRProvider`
/// at launch time so the Qwen3 backend is wired in only where its
/// large MLX dependency is also linked.
public protocol ASRServiceProvider: Sendable {
var backend: LocalASRBackend { get }
func make() -> ASRService
}
/// Built-in provider for the iOS SpeechAnalyzer path. Always present.
struct SpeechAnalyzerProvider: ASRServiceProvider {
let backend: LocalASRBackend = .speechAnalyzer
func make() -> ASRService { SpeechAnalyzerASR() }
}
// MARK: - PCM format conversion (testable helpers)
//
// Extracted from the audio-thread hot path so the scaling + clipping
@@ -115,6 +228,114 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
private var analyzer: SpeechAnalyzer?
private var analyzerTask: Task<Void, Never>?
private var analyzerFinished = false
/// Reused across pipelined chunks within one utterance (assets + format).
private var chunkPreparedLocaleID: String?
private var chunkAnalyzerFormat: AVAudioFormat?
func resetForNewUtterance() {
lock.withLock {
chunkPreparedLocaleID = nil
chunkAnalyzerFormat = nil
}
}
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
guard !samples.isEmpty else { return .success("") }
if Task.isCancelled { return .cancelled }
do {
let text = try await transcribeSamples(samples, locale: locale, reuseChunkPrep: true)
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? .success("") : .success(trimmed)
} catch is CancellationError {
return .cancelled
} catch {
return .failure(error.localizedDescription)
}
}
/// Analyze a single PCM buffer without the streaming `transcribe` wrapper.
private func transcribeSamples(
_ samples: [Float],
locale: Locale,
reuseChunkPrep: Bool
) async throws -> String {
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
throw ASRChunkError.localeUnsupported
}
let localeID = resolvedLocale.identifier(.bcp47)
let transcriber = DictationTranscriber(
locale: resolvedLocale,
preset: .progressiveLongDictation
)
let analyzerFormat: AVAudioFormat
let cachedPrep = lock.withLock { (chunkPreparedLocaleID, chunkAnalyzerFormat) }
if reuseChunkPrep,
cachedPrep.0 == localeID,
let cached = cachedPrep.1 {
analyzerFormat = cached
} else {
try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
compatibleWith: [transcriber],
considering: Self.captureFormat
) else {
throw ASRChunkError.formatUnsupported
}
analyzerFormat = format
lock.withLock {
chunkPreparedLocaleID = localeID
chunkAnalyzerFormat = format
}
}
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
guard let pcm = Self.makeAnalyzerPCMBuffer(from: snapshot, format: analyzerFormat) else {
throw ASRChunkError.formatUnsupported
}
let analyzer = SpeechAnalyzer(modules: [transcriber])
try await analyzer.prepareToAnalyze(in: analyzerFormat)
let resultsTask = Task<String, Error> {
var accumulator = ProgressiveDictationTranscriptAccumulator()
for try await result in transcriber.results {
if Task.isCancelled { break }
let text = String(result.text.characters)
_ = accumulator.ingest(range: result.range, text: text)
}
return accumulator.finalize()
}
let inputStream = AsyncStream<AnalyzerInput> { continuation in
continuation.yield(AnalyzerInput(buffer: pcm))
continuation.finish()
}
let lastSampleTime = try await analyzer.analyzeSequence(inputStream)
if let lastSampleTime {
try await analyzer.finalizeAndFinish(through: lastSampleTime)
} else {
await analyzer.cancelAndFinishNow()
}
return try await resultsTask.value
}
private enum ASRChunkError: LocalizedError {
case localeUnsupported
case formatUnsupported
var errorDescription: String? {
switch self {
case .localeUnsupported:
return SharedL10n.string("error.asr.localeUnsupported")
case .formatUnsupported:
return SharedL10n.string("error.asr.formatUnsupported")
}
}
}
/// Canonical capture format: 16 kHz mono Float32 from `AudioCaptureService`
/// / `PreviewASRController` before it reaches SpeechAnalyzer.
@@ -146,16 +367,21 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
do {
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
Self.debug("locale unsupported: \(locale.identifier(.bcp47))")
continuation.yield(.error("当前系统未分配可用语音语言模型,请稍后重试或切换语言"))
continuation.yield(.error(SharedL10n.string("error.asr.localeUnsupported")))
continuation.finish()
return
}
let transcriber = DictationTranscriber(locale: resolvedLocale, preset: .progressiveShortDictation)
// Each pipelined chunk is 30 s; long dictation preset keeps a
// single chunk coherent (Flow utterances run up to 3 min).
let transcriber = DictationTranscriber(
locale: resolvedLocale,
preset: .progressiveLongDictation
)
do {
try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
} catch {
Self.debug("asset prepare failed: \(error.localizedDescription)")
continuation.yield(.error("语音语言资源未就绪,请稍后重试"))
continuation.yield(.error(SharedL10n.string("error.asr.assetsNotReady")))
continuation.finish()
return
}
@@ -167,7 +393,7 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
compatibleWith: [transcriber],
considering: Self.captureFormat
) else {
continuation.yield(.error("当前设备不支持该语音输入格式"))
continuation.yield(.error(SharedL10n.string("error.asr.formatUnsupported")))
continuation.finish()
return
}
@@ -179,15 +405,16 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
// Apple recommends consuming `transcriber.results` concurrently
// while `analyzeSequence` drains the input stream.
let resultsTask = Task<String, Error> {
var lastText = ""
var accumulator = ProgressiveDictationTranscriptAccumulator()
for try await result in transcriber.results {
if Task.isCancelled { break }
let text = String(result.text.characters)
guard !text.isEmpty, text != lastText else { continue }
lastText = text
continuation.yield(.partial(text))
guard let full = accumulator.ingest(range: result.range, text: text) else {
continue
}
continuation.yield(.partial(full))
}
return lastText
return accumulator.finalize()
}
let lastSampleTime = try await newAnalyzer.analyzeSequence(inputStream)
@@ -195,7 +422,7 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
if let lastSampleTime {
try await newAnalyzer.finalizeAndFinish(through: lastSampleTime)
} else {
try await newAnalyzer.cancelAndFinishNow()
await newAnalyzer.cancelAndFinishNow()
}
let lastText: String
@@ -210,7 +437,7 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
let trimmed = lastText.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty {
continuation.yield(.error("未识别到语音内容,请重试"))
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
} else {
continuation.yield(.final(trimmed))
}
+24 -1
View File
@@ -35,6 +35,8 @@ public struct AppGroupStore: @unchecked Sendable {
static let modeId = "config.modeId"
static let localeId = "config.localeId"
static let engineMode = "config.engineMode"
static let localASRBackend = "config.localASRBackend"
static let uiLanguage = "config.uiLanguage"
}
// MARK: - Reads
@@ -70,12 +72,25 @@ public struct AppGroupStore: @unchecked Sendable {
defaults.string(forKey: Key.localeId) ?? "auto"
}
/// "local" on-device ASR only, no LLM polishing.
/// "local" on-device ASR only (raw transcript delivery).
/// "cloud" ASR + LLM polish (default behaviour).
public var engineMode: String {
defaults.string(forKey: Key.engineMode) ?? "cloud"
}
/// Which on-device ASR engine backs the "local" engine mode. Falls
/// back to the iOS SpeechAnalyzer path so legacy installs (which
/// never wrote this key) keep working.
public var localASRBackend: LocalASRBackend {
let raw = defaults.string(forKey: Key.localASRBackend) ?? LocalASRBackend.speechAnalyzer.rawValue
return LocalASRBackend(rawValue: raw) ?? .speechAnalyzer
}
/// Host-app UI language override (`auto` / `en` / `zh-Hans`).
public var uiLanguage: AppUILanguage {
AppUILanguage.fromStored(defaults.string(forKey: Key.uiLanguage))
}
// MARK: - Writes
public func setModeId(_ id: String) {
@@ -90,6 +105,14 @@ public struct AppGroupStore: @unchecked Sendable {
defaults.set(mode, forKey: Key.engineMode)
}
public func setLocalASRBackend(_ backend: LocalASRBackend) {
defaults.set(backend.rawValue, forKey: Key.localASRBackend)
}
public func setUILanguage(_ language: AppUILanguage) {
defaults.set(language.rawValue, forKey: Key.uiLanguage)
}
// MARK: - Client
public func makeClient() -> LLMClient {
@@ -0,0 +1,160 @@
// ChunkedUtterancePipeline.swift
// OSGKeyboard · Shared
//
// Pipelined Flow utterance ASR: split PCM while recording, transcribe chunks
// serially on a background queue, stitch partials for display and delivery.
import Foundation
public struct ChunkedUtteranceSuccess: Sendable, Equatable {
public let text: String
/// Non-fatal per-chunk ASR issues (delivered as soft warning when non-empty).
public let chunkWarnings: [String]
public init(text: String, chunkWarnings: [String] = []) {
self.text = text
self.chunkWarnings = chunkWarnings
}
}
public enum ChunkedUtterancePipelineOutcome: Sendable, Equatable {
case success(ChunkedUtteranceSuccess)
case failure(String)
case cancelled
}
/// Thread-safe queue between the chunk feeder and ASR worker.
private actor ChunkWorkQueue {
private var items: [UtteranceAudioChunk] = []
private var finished = false
private var waiters: [CheckedContinuation<UtteranceAudioChunk?, Never>] = []
func enqueue(_ chunk: UtteranceAudioChunk) {
items.append(chunk)
resumeWaiters()
}
func markFinished() {
finished = true
resumeWaiters()
}
func dequeue() async -> UtteranceAudioChunk? {
if !items.isEmpty {
return items.removeFirst()
}
if finished {
return nil
}
return await withCheckedContinuation { continuation in
waiters.append(continuation)
}
}
private func resumeWaiters() {
while !waiters.isEmpty {
if !items.isEmpty {
let waiter = waiters.removeFirst()
waiter.resume(returning: items.removeFirst())
} else if finished {
let waiter = waiters.removeFirst()
waiter.resume(returning: nil)
} else {
break
}
}
}
}
public actor ChunkedUtterancePipeline {
private let asr: ASRService
private let locale: Locale
private let config: FlowUtteranceChunkConfig
private var cancelled = false
public init(
asr: ASRService,
locale: Locale,
config: FlowUtteranceChunkConfig = .flowDefault
) {
self.asr = asr
self.locale = locale
self.config = config
}
public func cancel() {
cancelled = true
asr.cancel()
}
/// Consume `stream` until finished; ASR runs off the caller's actor while recording continues.
public func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
onPartial: @Sendable @escaping (String) -> Void
) async -> ChunkedUtterancePipelineOutcome {
asr.resetForNewUtterance()
let queue = ChunkWorkQueue()
var stitcher = UtteranceTranscriptStitcher()
var chunkWarnings: [String] = []
var failedChunks = 0
var processedChunks = 0
let feeder = Task {
for await chunk in UtteranceStreamChunker.chunks(from: stream, config: config) {
if Task.isCancelled { break }
await queue.enqueue(chunk)
}
await queue.markFinished()
}
while true {
if cancelled || Task.isCancelled {
feeder.cancel()
return .cancelled
}
guard let chunk = await queue.dequeue() else { break }
processedChunks += 1
let asr = self.asr
let locale = self.locale
let result = await Task.detached(priority: .userInitiated) {
await asr.transcribeChunk(samples: chunk.samples, locale: locale)
}.value
switch result {
case .success(let text):
stitcher.append(index: chunk.index, text: text)
let partial = stitcher.composed()
if !partial.isEmpty {
onPartial(partial)
}
case .failure(let message):
failedChunks += 1
chunkWarnings.append(
SharedL10n.format(
"error.asr.chunkFailed",
chunk.index + 1,
message
)
)
case .cancelled:
feeder.cancel()
return .cancelled
}
}
_ = await feeder.value
let finalText = stitcher.composed().trimmingCharacters(in: .whitespacesAndNewlines)
if finalText.isEmpty {
if failedChunks > 0, processedChunks == failedChunks {
return .failure(SharedL10n.string("error.asr.noSpeech"))
}
return .failure(SharedL10n.string("error.asr.noSpeech"))
}
return .success(ChunkedUtteranceSuccess(text: finalText, chunkWarnings: chunkWarnings))
}
}
@@ -21,6 +21,7 @@ public enum DictationBridge {
private enum Key {
static let pendingText = "dictation.pendingText"
static let polishWarning = "dictation.polishWarning"
static let updatedAt = "dictation.updatedAt"
static let status = "dictation.status"
static let statusUpdatedAt = "dictation.statusUpdatedAt"
@@ -67,12 +68,21 @@ public enum DictationBridge {
}
/// Store a transcript for the keyboard extension to consume.
public static func storePendingTranscript(_ text: String, defaults: UserDefaults? = nil) {
public static func storePendingTranscript(
_ text: String,
polishWarning: String? = nil,
defaults: UserDefaults? = nil
) {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
let store = resolvedDefaults(defaults)
store.set(trimmed, forKey: Key.pendingText)
store.set(Date().timeIntervalSince1970, forKey: Key.updatedAt)
if let polishWarning, !polishWarning.isEmpty {
store.set(polishWarning, forKey: Key.polishWarning)
} else {
store.removeObject(forKey: Key.polishWarning)
}
setStatus(.done, defaults: store)
}
@@ -81,6 +91,15 @@ public enum DictationBridge {
maxAge: TimeInterval = 180,
defaults: UserDefaults? = nil
) -> String? {
consumePendingDelivery(maxAge: maxAge, defaults: defaults)?.text
}
/// Returns and clears the pending delivery (text + optional polish
/// warning) if present.
public static func consumePendingDelivery(
maxAge: TimeInterval = 180,
defaults: UserDefaults? = nil
) -> TranscriptionDelivery? {
let store = resolvedDefaults(defaults)
guard let text = store.string(forKey: Key.pendingText) else {
return nil
@@ -92,14 +111,18 @@ public enum DictationBridge {
return nil
}
}
let warning = store.string(forKey: Key.polishWarning)
store.removeObject(forKey: Key.pendingText)
store.removeObject(forKey: Key.polishWarning)
store.removeObject(forKey: Key.updatedAt)
setStatus(.idle, defaults: store)
return text
return TranscriptionDelivery(text: text, polishWarning: warning)
}
public static func clear(defaults: UserDefaults? = nil) {
let store = resolvedDefaults(defaults)
store.removeObject(forKey: Key.pendingText)
store.removeObject(forKey: Key.polishWarning)
store.removeObject(forKey: Key.updatedAt)
setStatus(.idle, defaults: store)
}
@@ -0,0 +1,39 @@
// FlowAppLifecycle.swift
// OSGKeyboard · Shared
//
// Tracks whether the host app process is in the foreground.
// Retained for any future GPU-backed paths; CoreML ASR does not require it.
import Foundation
public final class FlowAppLifecycle: @unchecked Sendable {
public static let shared = FlowAppLifecycle()
private let lock = NSLock()
private var isForeground = true
private init() {}
/// `true` when the host app scene is active (`.active`).
public var allowsGPUInference: Bool {
lock.lock()
defer { lock.unlock() }
return isForeground
}
public func setForeground(_ foreground: Bool) {
lock.lock()
isForeground = foreground
lock.unlock()
}
/// Blocks until foreground or cancellation.
public func waitUntilForeground() async -> Bool {
while !allowsGPUInference {
if Task.isCancelled { return false }
try? await Task.sleep(nanoseconds: 200_000_000)
}
return true
}
}
@@ -34,7 +34,7 @@ private final class FlowCaptureStreamRelay: @unchecked Sendable {
}
func yield(_ snapshot: AudioBufferSnapshot) {
lock.withLock { continuation?.yield(snapshot) }
_ = lock.withLock { continuation?.yield(snapshot) }
}
func finish() {
@@ -250,6 +250,22 @@ public final class FlowContinuousCapture {
)
}
/// Re-activate capture after returning from background without
/// reinstalling the tap (iOS may deactivate the audio session).
public func reassertIfRunning() {
guard isRunning else { return }
let session = AVAudioSession.sharedInstance()
try? session.setCategory(
.playAndRecord,
mode: .measurement,
options: [.defaultToSpeaker, .allowBluetoothHFP, .mixWithOthers]
)
try? session.setActive(true, options: .notifyOthersOnDeactivation)
if !audioEngine.isRunning {
try? audioEngine.start()
}
}
/// Begin forwarding downsampled buffers to ASR for one utterance.
public func beginUtterance() -> AsyncStream<AudioBufferSnapshot> {
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
@@ -65,12 +65,24 @@ public enum FlowSessionBridge {
// MARK: - Session validity (keyboard)
/// True when expires is in the future and heartbeat is fresh.
/// True when the session contract is still valid (not expired).
/// Does not require a fresh heartbeat the host may be suspended in
/// background while the continuous audio session is frozen.
public static func isSessionActive(defaults: UserDefaults? = nil) -> Bool {
let store = resolvedDefaults(defaults)
flush(store)
guard store.bool(forKey: FlowSessionKeys.flowSessionActive) else { return false }
let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires)
guard expires > Date().timeIntervalSince1970 else { return false }
return expires > Date().timeIntervalSince1970
}
/// True when the host app recently wrote a heartbeat (foreground or
/// actively processing). Used for auto-start heuristics, not gating record.
public static func isHostReachable(defaults: UserDefaults? = nil) -> Bool {
let store = resolvedDefaults(defaults)
flush(store)
guard isSessionActive(defaults: store) else { return false }
let heartbeat = store.double(forKey: FlowSessionKeys.flowHeartbeat)
guard heartbeat > 0 else { return false }
@@ -124,6 +136,7 @@ public enum FlowSessionBridge {
public static func storeTranscriptionResult(
_ text: String,
polishWarning: String? = nil,
defaults: UserDefaults? = nil
) {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -131,6 +144,11 @@ public enum FlowSessionBridge {
let store = resolvedDefaults(defaults)
store.set(trimmed, forKey: FlowSessionKeys.transcriptionResult)
store.removeObject(forKey: FlowSessionKeys.transcriptionError)
if let polishWarning, !polishWarning.isEmpty {
store.set(polishWarning, forKey: FlowSessionKeys.transcriptionPolishWarning)
} else {
store.removeObject(forKey: FlowSessionKeys.transcriptionPolishWarning)
}
setRecordingState(.idle, defaults: store)
flush(store)
}
@@ -147,14 +165,24 @@ public enum FlowSessionBridge {
/// Returns and clears a pending transcription result, if any.
public static func consumeTranscriptionResult(defaults: UserDefaults? = nil) -> String? {
consumeTranscriptionDelivery(defaults: defaults)?.text
}
/// Returns and clears a pending transcription delivery (text + optional
/// polish warning), if any.
public static func consumeTranscriptionDelivery(
defaults: UserDefaults? = nil
) -> TranscriptionDelivery? {
let store = resolvedDefaults(defaults)
flush(store)
guard let text = store.string(forKey: FlowSessionKeys.transcriptionResult), !text.isEmpty else {
return nil
}
let warning = store.string(forKey: FlowSessionKeys.transcriptionPolishWarning)
store.removeObject(forKey: FlowSessionKeys.transcriptionResult)
store.removeObject(forKey: FlowSessionKeys.transcriptionPolishWarning)
flush(store)
return text
return TranscriptionDelivery(text: text, polishWarning: warning)
}
/// Returns and clears a pending transcription error, if any.
@@ -212,6 +240,7 @@ public enum FlowSessionBridge {
private static func clearTranscription(defaults: UserDefaults) {
defaults.removeObject(forKey: FlowSessionKeys.transcriptionResult)
defaults.removeObject(forKey: FlowSessionKeys.transcriptionPolishWarning)
defaults.removeObject(forKey: FlowSessionKeys.transcriptionError)
}
}
@@ -13,17 +13,39 @@ public enum FlowSessionKeys {
public static let keyboardRecordingState = "flow.keyboardRecordingState"
public static let transcriptionLanguage = "flow.transcriptionLanguage"
public static let transcriptionResult = "flow.transcriptionResult"
/// Soft warning when polish failed but raw transcript was delivered.
public static let transcriptionPolishWarning = "flow.transcriptionPolishWarning"
public static let transcriptionError = "flow.transcriptionError"
public static let audioLevels = "flow.audioLevels"
/// Heartbeat older than this implies the host app was killed.
/// Heartbeat older than this while the host is foreground likely killed.
public static let heartbeatStaleInterval: TimeInterval = 3
/// Default Flow session length when started from the keyboard.
public static let defaultSessionDuration: TimeInterval = 480
/// Maximum duration for a single keyboard utterance.
public static let maxUtteranceDuration: TimeInterval = 60
/// Maximum duration for a single keyboard utterance (3 minutes).
public static let maxUtteranceDuration: TimeInterval = 180
/// Host polls for pipelined ASR drain after mic stop. Pipelining usually
/// finishes most chunks during recording; this is a soft deadline before
/// blocking on `asrTask.value` (which waits until the pipeline exits).
public static let localASRWaitTimeout: TimeInterval = 120
public static let localQwen3ASRWaitTimeout: TimeInterval = 180
public static let cloudASRWaitTimeout: TimeInterval = 120
/// Keyboard watchdog after the user stops recording (not utterance max length).
/// Must cover worst-case post-stop backlog: remaining MLX/SpeechAnalyzer chunks
/// plus cloud LLM polish (see `PolishingService.effectiveTimeout` cap).
public static func keyboardResultTimeout(
engineMode: String,
localASRBackend: LocalASRBackend
) -> TimeInterval {
if engineMode == "local" {
return localASRBackend == .qwen3ASR ? 240 : 180
}
return 240
}
public enum RecordingState: String, Sendable, Equatable {
case idle
+11 -1
View File
@@ -71,8 +71,17 @@ public final class KeyboardState: ObservableObject {
@Published public var utteranceRemainingSeconds: Int = Int(FlowSessionKeys.maxUtteranceDuration)
/// Whether the host app's Flow voice session is currently valid.
@Published public var flowSessionActive: Bool = false
/// "local" ASR only, no LLM. "cloud" ASR + optional LLM polish.
/// "local" on-device ASR only. "cloud" ASR + LLM polish.
@Published public var engineMode: String = "cloud"
/// Which on-device ASR engine to use when `engineMode == "local"`.
/// Mirrored from `ProviderConfig.localASRBackend` for UI display
/// and for `state` consumers that want a single source of truth.
@Published public var localASRBackend: LocalASRBackend = .speechAnalyzer
/// `false` when the local engine needs on-device models that are
/// not yet downloaded (mirrored from App Group by the extension).
@Published public var localModelsReady: Bool = true
/// `true` when host app has preloaded Qwen weights into memory.
@Published public var localModelsLoaded: Bool = false
/// Convenience shorthand used by the pipeline and views.
public var isLocalEngine: Bool { engineMode == "local" }
@@ -86,6 +95,7 @@ public final class KeyboardState: ObservableObject {
public var setMode: (InputMode) -> Void = { _ in }
public var setLocale: (String) -> Void = { _ in }
public var setEngineMode: (String) -> Void = { _ in }
public var setLocalASRBackend: (LocalASRBackend) -> Void = { _ in }
public var insertNewline: () -> Void = {}
public var insertSpace: () -> Void = {}
public var deleteBackward: () -> Void = {}
+14 -7
View File
@@ -17,13 +17,20 @@ public enum LLMError: Error, LocalizedError, Sendable, Equatable {
public var errorDescription: String? {
switch self {
case .invalidURL: return "API 地址无效。请在设置中检查 Base URL。"
case .noAPIKey: return "未填写 API Key。"
case .http(let s): return "API 返回 HTTP \(s)。请稍后重试或联系服务方。"
case .decoding: return "解析 API 响应失败。"
case .transport: return "网络错误,请检查连接后重试。"
case .rateLimited: return "API 调用过于频繁,请稍候再试。"
case .cancelled: return "请求已取消。"
case .invalidURL:
return SharedL10n.string("error.llm.invalidURL")
case .noAPIKey:
return SharedL10n.string("error.llm.noAPIKey")
case .http(let status):
return SharedL10n.format("error.llm.http", status)
case .decoding:
return SharedL10n.string("error.llm.decoding")
case .transport:
return SharedL10n.string("error.llm.transport")
case .rateLimited:
return SharedL10n.string("error.llm.rateLimited")
case .cancelled:
return SharedL10n.string("error.llm.cancelled")
}
}
}
@@ -36,7 +36,7 @@ private final class CaptureStreamRelay: @unchecked Sendable {
}
func yield(_ snapshot: AudioBufferSnapshot) {
lock.withLock { continuation?.yield(snapshot) }
_ = lock.withLock { continuation?.yield(snapshot) }
}
func finish() {
@@ -73,7 +73,7 @@ public final class LiveDictationController: ObservableObject {
/// next recording starts from zero.
@Published public var lastFinal: String = ""
private let asr: ASRService = ASRServiceFactory.make()
private let asr: ASRService
private let audioEngine = AVAudioEngine()
/// `internal` (not `private`) so the regression test in
/// `OSGKeyboardTests/PreviewASRControllerStateTests.swift` can
@@ -83,10 +83,20 @@ public final class LiveDictationController: ObservableObject {
/// code outside the class from racing on it.
public var asrTask: Task<Void, Never>?
private let streamRelay = CaptureStreamRelay()
private var chunkedPipeline: ChunkedUtterancePipeline?
private var didConfigureAudioSession = false
private var didInstallTap = false
public init() {}
public init(asr: ASRService? = nil) {
// Resolve through the factory so the user's `LocalASRBackend`
// selection is honoured. Tests can pass a stub `asr` directly
// to bypass the factory and exercise the controller in
// isolation.
self.asr = asr ?? ASRServiceFactory.make(
engineMode: ProviderConfig.shared.engineMode,
localBackend: ProviderConfig.shared.localASRBackend
)
}
/// Start dictation using a persisted settings locale id (`auto`, `zh-Hans`, ).
public func start(localeId: String) async {
@@ -112,6 +122,10 @@ public final class LiveDictationController: ObservableObject {
// same `events` stream.
asrTask?.cancel()
asrTask = nil
if let pipeline = chunkedPipeline {
Task { await pipeline.cancel() }
}
chunkedPipeline = nil
teardownCapturePipeline()
phase = .requestingPermission
currentPartial = ""
@@ -335,29 +349,36 @@ public final class LiveDictationController: ObservableObject {
return
}
// 5. Wire up ASR.
let events = asr.transcribe(
stream: stream,
locale: locale
)
asrTask = Task { @MainActor [weak self] in
guard let self else { return }
for await event in events {
switch event {
case .capability:
break
case .partial(let s):
self.currentPartial = s
case .final(let s):
let trimmed = s.trimmingCharacters(in: .whitespacesAndNewlines)
self.lastFinal = trimmed
self.currentPartial = ""
self.phase = .idle
case .error(let m):
self.debug("asr error: \(m)")
self.teardownCapturePipeline()
self.errorMessage = m
self.phase = .error(m)
// 5. Pipelined ASR (same chunk path as Flow host).
let pipeline = ChunkedUtterancePipeline(asr: asr, locale: locale)
chunkedPipeline = pipeline
asrTask = Task.detached(priority: .userInitiated) { [weak controller = self] in
let outcome = await pipeline.transcribe(stream: stream) { partial in
Task { @MainActor in
controller?.currentPartial = partial
}
}
await MainActor.run {
guard let controller else { return }
switch outcome {
case .success(let success):
let trimmed = success.text.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty {
controller.lastFinal = trimmed
controller.currentPartial = ""
}
if controller.phase == .processing || controller.phase == .recording {
controller.phase = .idle
}
case .failure(let message):
controller.debug("asr error: \(message)")
controller.teardownCapturePipeline()
controller.errorMessage = message
controller.phase = .error(message)
case .cancelled:
if controller.phase == .processing {
controller.phase = .idle
}
}
}
}
@@ -0,0 +1,104 @@
// OnDeviceModelStatus.swift
// OSGKeyboard · Shared
//
// Mirrors on-device model download state into the App Group so the
// keyboard extension can show readiness hints without reading the
// host app's Caches directory.
import Foundation
public enum OnDeviceModelStatus {
private enum Key {
static func downloaded(_ model: OnDeviceModel) -> String {
"models.\(model.rawValue).downloaded"
}
static func progress(_ model: OnDeviceModel) -> String {
"models.\(model.rawValue).downloadProgress"
}
static let modelsLoadedInMemory = "models.loadedInMemory"
}
// MARK: - Writes (host app)
public static func setDownloaded(_ downloaded: Bool, for model: OnDeviceModel) {
guard AppGroup.isAvailable else { return }
AppGroup.defaults.set(downloaded, forKey: Key.downloaded(model))
if downloaded {
clearProgress(for: model)
}
}
public static func setProgress(_ progress: Double?, for model: OnDeviceModel) {
guard AppGroup.isAvailable else { return }
if let progress {
AppGroup.defaults.set(progress, forKey: Key.progress(model))
} else {
AppGroup.defaults.removeObject(forKey: Key.progress(model))
}
}
public static func clearProgress(for model: OnDeviceModel) {
guard AppGroup.isAvailable else { return }
AppGroup.defaults.removeObject(forKey: Key.progress(model))
}
public static func setModelsLoadedInMemory(_ loaded: Bool) {
guard AppGroup.isAvailable else { return }
AppGroup.defaults.set(loaded, forKey: Key.modelsLoadedInMemory)
}
public static func modelsLoadedInMemory(defaults: UserDefaults? = nil) -> Bool {
let store = defaults ?? (AppGroup.isAvailable ? AppGroup.defaults : .standard)
return store.bool(forKey: Key.modelsLoadedInMemory)
}
// MARK: - Reads (keyboard + host app)
public static func isDownloaded(_ model: OnDeviceModel, defaults: UserDefaults? = nil) -> Bool {
let store = defaults ?? (AppGroup.isAvailable ? AppGroup.defaults : .standard)
return store.bool(forKey: Key.downloaded(model))
}
public static func downloadProgress(_ model: OnDeviceModel, defaults: UserDefaults? = nil) -> Double? {
let store = defaults ?? (AppGroup.isAvailable ? AppGroup.defaults : .standard)
guard store.object(forKey: Key.progress(model)) != nil else { return nil }
return store.double(forKey: Key.progress(model))
}
/// Whether the currently selected local-engine stack has every
/// required on-device model downloaded.
public static func isLocalStackReady(
asrBackend: LocalASRBackend,
defaults: UserDefaults? = nil
) -> Bool {
if asrBackend == .qwen3ASR {
return isDownloaded(.qwen3ASR, defaults: defaults)
}
return true
}
/// First missing model for the active local stack, if any.
public static func firstMissingModel(
asrBackend: LocalASRBackend,
defaults: UserDefaults? = nil
) -> OnDeviceModel? {
if asrBackend == .qwen3ASR, !isDownloaded(.qwen3ASR, defaults: defaults) {
return .qwen3ASR
}
return nil
}
}
// MARK: - On-device Qwen3 runtime
/// CoreML ASR requires iOS 18+ / macOS 15+ (MLState KV cache).
public enum OnDeviceMLRuntime {
/// Whether Qwen3-ASR CoreML can run in this process.
public static var supportsOnDeviceQwen3: Bool {
if #available(iOS 18.0, *) {
return true
}
return false
}
}
@@ -5,9 +5,9 @@
// to produce polished, well-punctuated text. Falls back to the raw transcript
// if the LLM call fails or times out.
//
// Mode-aware: when `modeId == "off"` the service short-circuits and returns
// the trimmed input without touching the network. This is the runtime
// guarantee behind the keyboard's "Off · " mode.
// Cloud engine always runs the LLM polish step (settings no longer expose
// off / transcribe). Local engine (`engineMode == "local"`) is ASR-only
// the raw transcript is returned unchanged and cloud API settings are ignored.
import Foundation
@@ -16,7 +16,6 @@ public actor PolishingService {
public enum PolishError: Error, Equatable {
case noTranscript
case timeout
case modeOff
}
private let store: AppGroupStore
@@ -44,25 +43,25 @@ public actor PolishingService {
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { throw PolishError.noTranscript }
// Mode-aware short-circuit. When the user has selected "Off", the
// keyboard must never hit the network we return the trimmed
// input as-is. This is the same value the view controller would
// produce if it skipped `polish()` entirely, but having the
// guarantee at the service layer means future call sites (CLI,
// tests, alternate keyboards) inherit it for free.
if store.modeId == "off" {
// Local engine: ASR-only no on-device or cloud polish.
if store.engineMode == "local" {
return trimmed
}
return try await polishRemote(trimmed)
}
private func polishRemote(_ trimmed: String) async throws -> String {
let client = injectedClient ?? store.makeClient()
let prompt = store.systemPrompt
let budget = effectiveTimeout(for: trimmed)
return try await withThrowingTaskGroup(of: String.self) { group in
group.addTask {
try await client.polish(trimmed, systemPrompt: prompt)
}
group.addTask {
try await Task.sleep(nanoseconds: UInt64(self.timeout * 1_000_000_000))
try await Task.sleep(nanoseconds: UInt64(budget * 1_000_000_000))
throw PolishError.timeout
}
let result = try await group.next()!
@@ -70,4 +69,10 @@ public actor PolishingService {
return result
}
}
}
/// Scale polish budget with transcript length (3-minute Flow utterances).
private func effectiveTimeout(for text: String) -> TimeInterval {
let scaled = timeout + (Double(text.count) / 200.0) * 2.0
return min(max(scaled, timeout), 120)
}
}
@@ -0,0 +1,64 @@
// ProgressiveDictationTranscriptAccumulator.swift
// OSGKeyboard · Shared
//
// Merges progressive `DictationTranscriber` results into one transcript.
// Short-form presets may emit a new time range after ~30 s; treating the
// latest partial as the full transcript drops earlier segments.
import Foundation
import CoreMedia
/// Combines volatile partials and finalized segments from
/// `DictationTranscriber.results` into a single growing transcript.
public struct ProgressiveDictationTranscriptAccumulator: Sendable {
private struct Segment: Sendable {
let startSeconds: Double
var text: String
}
private var segments: [Segment] = []
private var lastEmitted = ""
public init() {}
/// Ingest one analyzer result. Returns a non-nil full transcript when the
/// composed text changed since the previous emission.
public mutating func ingest(range: CMTimeRange, text: String) -> String? {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let start = range.start.seconds
if let idx = segments.lastIndex(where: { abs($0.startSeconds - start) < 0.001 }) {
// Same audio window volatile refinement of the current segment.
segments[idx].text = trimmed
} else if let last = segments.last,
trimmed.hasPrefix(last.text) || last.text.hasPrefix(trimmed) {
// Cumulative progressive update without a range change.
let longer = trimmed.count >= last.text.count ? trimmed : last.text
segments[segments.count - 1].text = longer
} else {
// New time range append instead of replacing earlier speech.
segments.append(Segment(startSeconds: start, text: trimmed))
}
let full = composedText()
guard full != lastEmitted else { return nil }
lastEmitted = full
return full
}
/// Final composed transcript after the results stream finishes.
public mutating func finalize() -> String {
let full = composedText()
lastEmitted = full
return full
}
private func composedText() -> String {
segments.reduce(into: "") { partial, segment in
partial = DictationTextComposer.compose(anchor: partial, live: segment.text)
}
}
}
@@ -6,9 +6,12 @@
import Foundation
public enum ProviderDisplayName {
public static func name(for providerId: String) -> String {
public static func name(
for providerId: String,
language: AppUILanguage? = nil
) -> String {
let key = "provider.\(providerId)"
let localized = NSLocalizedString(key, comment: "")
let localized = SharedL10n.string(key, language: language)
if localized != key { return localized }
return LLMProvider.provider(id: providerId).name
}
@@ -0,0 +1,101 @@
// UtteranceStreamChunker.swift
// OSGKeyboard · Shared
//
// Splits a Flow utterance PCM stream into ASR-sized chunks. When possible,
// extends slightly past the max window to the next pause instead of cutting
// mid-word.
import Foundation
public enum UtteranceStreamChunker {
/// Yields chunks as audio arrives; the final chunk is marked `isLast`.
public static func chunks(
from stream: AsyncStream<AudioBufferSnapshot>,
config: FlowUtteranceChunkConfig = .flowDefault
) -> AsyncStream<UtteranceAudioChunk> {
AsyncStream { continuation in
let task = Task {
var buffer: [Float] = []
buffer.reserveCapacity(config.maxChunkSamples + config.pauseExtensionSamples)
var chunkIndex = 0
func emit(upTo splitEnd: Int, isLast: Bool) {
guard splitEnd > 0, splitEnd <= buffer.count else { return }
let chunkSamples = Array(buffer[..<splitEnd])
continuation.yield(
UtteranceAudioChunk(index: chunkIndex, samples: chunkSamples, isLast: isLast)
)
chunkIndex += 1
if splitEnd >= buffer.count {
buffer.removeAll(keepingCapacity: true)
} else {
let overlapStart = max(0, splitEnd - config.overlapSamples)
buffer = Array(buffer[overlapStart...])
}
}
for await snap in stream {
if Task.isCancelled { break }
guard !snap.samples.isEmpty else { continue }
buffer.append(contentsOf: snap.samples)
while buffer.count >= config.maxChunkSamples {
let split = pauseAwareSplitIndex(in: buffer, config: config)
emit(upTo: split, isLast: false)
}
}
if !buffer.isEmpty {
emit(upTo: buffer.count, isLast: true)
} else if chunkIndex == 0 {
// Empty utterance no chunks.
} else {
// Stream ended exactly on boundary; mark prior path complete.
}
continuation.finish()
}
continuation.onTermination = { _ in
task.cancel()
}
}
}
/// Pick a split index at or after `maxChunkSamples`, preferring a pause.
static func pauseAwareSplitIndex(
in buffer: [Float],
config: FlowUtteranceChunkConfig
) -> Int {
let minSplit = config.maxChunkSamples
guard buffer.count >= minSplit else { return buffer.count }
let searchEnd = min(buffer.count, minSplit + config.pauseExtensionSamples)
if searchEnd <= minSplit {
return minSplit
}
let windowSize = max(config.sampleRate / 50, 160) // ~20 ms
var bestPause: Int?
var idx = minSplit
while idx + windowSize <= searchEnd {
if rms(of: buffer, start: idx, count: windowSize) < config.pauseRMSThreshold {
bestPause = idx + windowSize
}
idx += windowSize / 2
}
return bestPause ?? minSplit
}
static func rms(of samples: [Float], start: Int, count: Int) -> Float {
guard start >= 0, count > 0, start + count <= samples.count else { return 1 }
var sum: Float = 0
for i in start..<(start + count) {
let v = samples[i]
sum += v * v
}
return sqrtf(sum / Float(count))
}
}
@@ -0,0 +1,109 @@
// UtteranceTranscriptStitcher.swift
// OSGKeyboard · Shared
//
// Orders pipelined chunk transcripts and merges overlap at boundaries.
import Foundation
public struct UtteranceTranscriptStitcher: Sendable {
private var segments: [(index: Int, text: String)] = []
public init() {}
public mutating func append(index: Int, text: String) {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
if let existing = segments.firstIndex(where: { $0.index == index }) {
segments[existing].text = trimmed
} else {
segments.append((index, trimmed))
segments.sort { $0.index < $1.index }
}
}
public func composed() -> String {
guard let first = segments.first else { return "" }
var result = first.text
for segment in segments.dropFirst() {
result = Self.mergeWithOverlap(previous: result, next: segment.text)
}
return result
}
/// Merge `next` onto `previous`, dropping duplicated suffix/prefix overlap.
public static func mergeWithOverlap(previous: String, next: String) -> String {
let trimmedNext = next.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmedNext.isEmpty else { return previous }
guard !previous.isEmpty else { return trimmedNext }
// Character-granular probe works for CJK without word boundaries.
let prevChars = Array(previous)
let nextChars = Array(trimmedNext)
let maxProbe = min(64, prevChars.count, nextChars.count)
if maxProbe > 0 {
for length in stride(from: maxProbe, through: 1, by: -1) {
let suffix = prevChars.suffix(length)
let prefix = nextChars.prefix(length)
if suffix.elementsEqual(prefix) {
return previous + String(nextChars.dropFirst(length))
}
}
}
// Punctuation-insensitive CJK overlap (e.g. "" + "").
let normalizedPrev = normalizeForOverlap(previous)
let normalizedNext = normalizeForOverlap(trimmedNext)
let nPrev = Array(normalizedPrev)
let nNext = Array(normalizedNext)
let normProbe = min(64, nPrev.count, nNext.count)
if normProbe > 0 {
for length in stride(from: normProbe, through: 2, by: -1) {
if nPrev.suffix(length).elementsEqual(nNext.prefix(length)) {
// Map normalized overlap length back to raw `next` drop count.
let drop = overlapDropCount(in: trimmedNext, normalizedPrefixLength: length)
return previous + String(trimmedNext.dropFirst(drop))
}
}
}
// English / spaced languages.
let maxWordProbe = min(6, previous.split(separator: " ").count, trimmedNext.split(separator: " ").count)
if maxWordProbe > 0 {
let prevWords = previous.split(separator: " ", omittingEmptySubsequences: true)
let nextWords = trimmedNext.split(separator: " ", omittingEmptySubsequences: true)
for wordCount in stride(from: maxWordProbe, through: 1, by: -1) {
if prevWords.suffix(wordCount).elementsEqual(nextWords.prefix(wordCount)) {
let mergedPrefix = nextWords.dropFirst(wordCount).joined(separator: " ")
if mergedPrefix.isEmpty { return previous }
if previous.last == " " || previous.last == "\n" {
return previous + mergedPrefix
}
return previous + " " + mergedPrefix
}
}
}
return DictationTextComposer.compose(anchor: previous, live: trimmedNext)
}
private static func normalizeForOverlap(_ text: String) -> String {
text.unicodeScalars.filter {
!CharacterSet.whitespacesAndNewlines.contains($0)
&& !CharacterSet.punctuationCharacters.contains($0)
}.map { Character($0) }.reduce(into: "") { $0.append($1) }
}
/// How many raw characters to drop from `next` given a normalized-prefix overlap length.
private static func overlapDropCount(in next: String, normalizedPrefixLength: Int) -> Int {
var normalizedCount = 0
var rawIndex = next.startIndex
while rawIndex < next.endIndex, normalizedCount < normalizedPrefixLength {
let scalar = next[rawIndex]
if !scalar.isWhitespace, !scalar.isPunctuation {
normalizedCount += 1
}
rawIndex = next.index(after: rawIndex)
}
return next.distance(from: next.startIndex, to: rawIndex)
}
}
+30
View File
@@ -0,0 +1,30 @@
/* Engine status labels */
"engine.summary.local" = "On-device · %@";
"engine.summary.cloud" = "Active: %@";
"engine.summary.cloudWithModel" = "Active: %1$@ · %2$@";
"engine.asr.appleSpeech" = "Apple SpeechAnalyzer";
"model.qwen3asr.name" = "Qwen3-ASR 0.6B (CoreML)";
/* LLM providers */
"provider.openai" = "OpenAI";
"provider.deepseek" = "DeepSeek";
"provider.qwen" = "Qwen (DashScope)";
"provider.zhipu" = "Zhipu GLM";
"provider.moonshot" = "Moonshot";
"provider.custom" = "Custom";
/* LLM errors */
"error.llm.invalidURL" = "Invalid API URL. Check Base URL in Settings.";
"error.llm.noAPIKey" = "API Key is missing.";
"error.llm.http" = "API returned HTTP %lld. Try again later or contact the provider.";
"error.llm.decoding" = "Failed to parse the API response.";
"error.llm.transport" = "Network error. Check your connection and try again.";
"error.llm.rateLimited" = "Too many API requests. Please wait and try again.";
"error.llm.cancelled" = "Request cancelled.";
/* ASR errors */
"error.asr.localeUnsupported" = "Speech language assets are unavailable. Try again later or switch the recognition language.";
"error.asr.assetsNotReady" = "Speech language assets are not ready. Try again later.";
"error.asr.formatUnsupported" = "This device does not support the required audio format.";
"error.asr.noSpeech" = "No speech detected. Please try again.";
"error.asr.chunkFailed" = "Segment %lld failed: %@";
@@ -0,0 +1,30 @@
/* Engine status labels */
"engine.summary.local" = "本地 · %@";
"engine.summary.cloud" = "当前:%@";
"engine.summary.cloudWithModel" = "当前:%1$@ · %2$@";
"engine.asr.appleSpeech" = "Apple 语音识别";
"model.qwen3asr.name" = "Qwen3-ASR 0.6B (CoreML)";
/* LLM providers */
"provider.openai" = "OpenAI";
"provider.deepseek" = "DeepSeek";
"provider.qwen" = "通义千问";
"provider.zhipu" = "智谱 GLM";
"provider.moonshot" = "月之暗面";
"provider.custom" = "自定义";
/* LLM errors */
"error.llm.invalidURL" = "API 地址无效。请在设置中检查 Base URL。";
"error.llm.noAPIKey" = "未填写 API Key。";
"error.llm.http" = "API 返回 HTTP %lld。请稍后重试或联系服务方。";
"error.llm.decoding" = "解析 API 响应失败。";
"error.llm.transport" = "网络错误,请检查连接后重试。";
"error.llm.rateLimited" = "API 调用过于频繁,请稍候再试。";
"error.llm.cancelled" = "请求已取消。";
/* ASR errors */
"error.asr.localeUnsupported" = "当前系统未分配可用语音语言模型,请稍后重试或切换语言。";
"error.asr.assetsNotReady" = "语音语言资源未就绪,请稍后重试。";
"error.asr.formatUnsupported" = "当前设备不支持该语音输入格式。";
"error.asr.noSpeech" = "未识别到语音内容,请重试。";
"error.asr.chunkFailed" = "第 %lld 段识别失败:%@";