feat: migrate on-device Qwen3 ASR to CoreML for background Flow dictation
Replace MLX GPU inference with CoreML bundles so transcription continues while the host app is backgrounded. Adds model download and warm-up, vendored Qwen3Speech, and updates onboarding, settings, and copy for the ~1.6 GB CoreML package (iOS 18+).
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
// SharedL10n.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Localized strings shipped inside the shared framework (Shared.strings).
|
||||
// Respects the in-app UI language override from App Group settings.
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum SharedL10n {
|
||||
private static let table = "Shared"
|
||||
private static let container = Bundle(for: SharedBundleToken.self)
|
||||
|
||||
public static func string(
|
||||
_ key: String,
|
||||
language: AppUILanguage? = nil
|
||||
) -> String {
|
||||
let lang = language ?? AppGroupStore().uiLanguage
|
||||
let bundle = AppUILanguage.localizedBundle(in: container, language: lang)
|
||||
return NSLocalizedString(
|
||||
key,
|
||||
tableName: table,
|
||||
bundle: bundle,
|
||||
value: key,
|
||||
comment: ""
|
||||
)
|
||||
}
|
||||
|
||||
public static func format(
|
||||
_ key: String,
|
||||
language: AppUILanguage? = nil,
|
||||
_ args: CVarArg...
|
||||
) -> String {
|
||||
String(
|
||||
format: string(key, language: language),
|
||||
locale: Locale.current,
|
||||
arguments: args
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private final class SharedBundleToken {}
|
||||
@@ -0,0 +1,87 @@
|
||||
// AppUILanguage.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// In-app UI language override (main app + keyboard extension strings).
|
||||
// Distinct from `localeId`, which controls speech recognition language.
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum AppUILanguage: String, CaseIterable, Identifiable, Sendable, Codable {
|
||||
case auto
|
||||
case english = "en"
|
||||
case chinese = "zh-Hans"
|
||||
|
||||
public var id: String { rawValue }
|
||||
|
||||
public var labelKey: String {
|
||||
switch self {
|
||||
case .auto: return "settings.appLanguage.auto"
|
||||
case .english: return "settings.appLanguage.english"
|
||||
case .chinese: return "settings.appLanguage.chinese"
|
||||
}
|
||||
}
|
||||
|
||||
/// Locale for SwiftUI `.environment(\.locale, …)` in the host app.
|
||||
public var swiftUILocale: Locale {
|
||||
switch self {
|
||||
case .auto:
|
||||
return Locale.autoupdatingCurrent
|
||||
case .english:
|
||||
return Locale(identifier: "en")
|
||||
case .chinese:
|
||||
return Locale(identifier: "zh-Hans")
|
||||
}
|
||||
}
|
||||
|
||||
/// `.lproj` folder name used for manual bundle lookups (extension).
|
||||
public func resolvedLanguageCode(
|
||||
preferredLanguages: [String] = Locale.preferredLanguages
|
||||
) -> String {
|
||||
switch self {
|
||||
case .english:
|
||||
return "en"
|
||||
case .chinese:
|
||||
return "zh-Hans"
|
||||
case .auto:
|
||||
if preferredLanguages.contains(where: { $0.hasPrefix("zh") }) {
|
||||
return "zh-Hans"
|
||||
}
|
||||
return "en"
|
||||
}
|
||||
}
|
||||
|
||||
public static func fromStored(_ raw: String?) -> AppUILanguage {
|
||||
guard let raw, let value = AppUILanguage(rawValue: raw) else { return .auto }
|
||||
return value
|
||||
}
|
||||
|
||||
/// Picks the best-matching `.lproj` inside `container` for this preference.
|
||||
public static func localizedBundle(
|
||||
in container: Bundle,
|
||||
language: AppUILanguage,
|
||||
preferredLanguages: [String] = Locale.preferredLanguages
|
||||
) -> Bundle {
|
||||
let code = language.resolvedLanguageCode(preferredLanguages: preferredLanguages)
|
||||
guard let path = container.path(forResource: code, ofType: "lproj"),
|
||||
let bundle = Bundle(path: path) else {
|
||||
return container
|
||||
}
|
||||
return bundle
|
||||
}
|
||||
|
||||
public static func localizedString(
|
||||
_ key: String,
|
||||
tableName: String?,
|
||||
bundle container: Bundle,
|
||||
language: AppUILanguage = AppGroupStore().uiLanguage
|
||||
) -> String {
|
||||
let bundle = localizedBundle(in: container, language: language)
|
||||
return NSLocalizedString(
|
||||
key,
|
||||
tableName: tableName,
|
||||
bundle: bundle,
|
||||
value: key,
|
||||
comment: ""
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -9,18 +9,37 @@ public enum EngineServiceLabel {
|
||||
public static func summary(
|
||||
engineMode: String,
|
||||
providerId: String,
|
||||
model: String
|
||||
model: String,
|
||||
localASRBackend: LocalASRBackend = .speechAnalyzer,
|
||||
language: AppUILanguage? = nil
|
||||
) -> String {
|
||||
let isChinese = Locale.preferredLanguages.first?.hasPrefix("zh") == true
|
||||
let prefix = isChinese ? "当前:" : "Active: "
|
||||
let lang = language ?? AppGroupStore().uiLanguage
|
||||
if engineMode == "local" {
|
||||
return isChinese
|
||||
? "\(prefix)本地引擎 · Apple SpeechAnalyzer"
|
||||
: "\(prefix)On-device · Apple SpeechAnalyzer"
|
||||
let asrName = asrDisplayName(for: localASRBackend, language: lang)
|
||||
return SharedL10n.format("engine.summary.local", language: lang, asrName)
|
||||
}
|
||||
let providerName = ProviderDisplayName.name(for: providerId)
|
||||
let providerName = ProviderDisplayName.name(for: providerId, language: lang)
|
||||
let trimmedModel = model.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if trimmedModel.isEmpty { return "\(prefix)\(providerName)" }
|
||||
return "\(prefix)\(providerName) · \(trimmedModel)"
|
||||
if trimmedModel.isEmpty {
|
||||
return SharedL10n.format("engine.summary.cloud", language: lang, providerName)
|
||||
}
|
||||
return SharedL10n.format(
|
||||
"engine.summary.cloudWithModel",
|
||||
language: lang,
|
||||
providerName,
|
||||
trimmedModel
|
||||
)
|
||||
}
|
||||
|
||||
private static func asrDisplayName(
|
||||
for backend: LocalASRBackend,
|
||||
language: AppUILanguage
|
||||
) -> String {
|
||||
switch backend {
|
||||
case .speechAnalyzer:
|
||||
return SharedL10n.string("engine.asr.appleSpeech", language: language)
|
||||
case .qwen3ASR:
|
||||
return SharedL10n.string("model.qwen3asr.name", language: language)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
// FlowUtteranceChunkConfig.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Chunking policy for pipelined Flow utterance ASR (up to 3 minutes).
|
||||
|
||||
import Foundation
|
||||
|
||||
public struct FlowUtteranceChunkConfig: Sendable, Equatable {
|
||||
/// Target maximum duration per ASR chunk.
|
||||
public let maxChunkDurationSeconds: TimeInterval
|
||||
/// Tail overlap fed into the next chunk for boundary dedup when stitching.
|
||||
public let overlapDurationSeconds: TimeInterval
|
||||
/// After hitting the max window, wait up to this long for a pause before hard-splitting.
|
||||
public let pauseExtensionMaxSeconds: TimeInterval
|
||||
/// RMS below this is treated as a pause candidate (Float32 mono @ 16 kHz).
|
||||
public let pauseRMSThreshold: Float
|
||||
public let sampleRate: Int
|
||||
|
||||
public init(
|
||||
maxChunkDurationSeconds: TimeInterval,
|
||||
overlapDurationSeconds: TimeInterval,
|
||||
pauseExtensionMaxSeconds: TimeInterval,
|
||||
pauseRMSThreshold: Float,
|
||||
sampleRate: Int
|
||||
) {
|
||||
self.maxChunkDurationSeconds = maxChunkDurationSeconds
|
||||
self.overlapDurationSeconds = overlapDurationSeconds
|
||||
self.pauseExtensionMaxSeconds = pauseExtensionMaxSeconds
|
||||
self.pauseRMSThreshold = pauseRMSThreshold
|
||||
self.sampleRate = sampleRate
|
||||
}
|
||||
|
||||
public var maxChunkSamples: Int {
|
||||
Int(maxChunkDurationSeconds * Double(sampleRate))
|
||||
}
|
||||
|
||||
public var overlapSamples: Int {
|
||||
Int(overlapDurationSeconds * Double(sampleRate))
|
||||
}
|
||||
|
||||
public var pauseExtensionSamples: Int {
|
||||
Int(pauseExtensionMaxSeconds * Double(sampleRate))
|
||||
}
|
||||
|
||||
/// Default for keyboard Flow utterances (≤ 3 min, pipelined ASR).
|
||||
public static let flowDefault = FlowUtteranceChunkConfig(
|
||||
maxChunkDurationSeconds: 30,
|
||||
overlapDurationSeconds: 0.5,
|
||||
pauseExtensionMaxSeconds: 2,
|
||||
pauseRMSThreshold: 0.015,
|
||||
sampleRate: 16_000
|
||||
)
|
||||
}
|
||||
|
||||
public struct UtteranceAudioChunk: Sendable, Equatable {
|
||||
public let index: Int
|
||||
public let samples: [Float]
|
||||
public let isLast: Bool
|
||||
|
||||
public init(index: Int, samples: [Float], isLast: Bool) {
|
||||
self.index = index
|
||||
self.samples = samples
|
||||
self.isLast = isLast
|
||||
}
|
||||
|
||||
public var durationSeconds: Double {
|
||||
Double(samples.count) / 16_000.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// LocalASRBackend.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Identifies which on-device speech recognition engine to use when the
|
||||
// user picks the "local" engine (no cloud LLM polish). The shared
|
||||
// factory `ASRServiceFactory` dispatches on this enum; the settings UI
|
||||
// renders it as a picker.
|
||||
//
|
||||
// Why an enum in `Shared` rather than living next to the concrete
|
||||
// `ASRService` implementations: the value must be serialisable into
|
||||
// the App Group store (so the keyboard extension can observe the
|
||||
// selection), exposed via `ProviderConfig` (UI binding) and consumed
|
||||
// by every layer that asks for an ASR backend.
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum LocalASRBackend: String, CaseIterable, Identifiable, Sendable, Codable {
|
||||
/// iOS 26 `SpeechAnalyzer` + `DictationTranscriber`. Always
|
||||
/// on-device, no asset download, ships with iOS. Default for every
|
||||
/// fresh install — anything else is opt-in.
|
||||
case speechAnalyzer
|
||||
|
||||
/// Qwen3-ASR-0.6B via CoreML (Neural Engine + CPU). Stronger on Chinese
|
||||
/// dialects and noisy audio than `SpeechAnalyzer`, works in Flow while
|
||||
/// the host app is backgrounded, but requires a ~1.6 GB download on first
|
||||
/// use and iOS 18+.
|
||||
case qwen3ASR
|
||||
|
||||
public var id: String { rawValue }
|
||||
|
||||
/// Localisation key for the human label in the settings picker.
|
||||
public var labelKey: String {
|
||||
switch self {
|
||||
case .speechAnalyzer: return "asr.backend.speechAnalyzer.label"
|
||||
case .qwen3ASR: return "asr.backend.qwen3.label"
|
||||
}
|
||||
}
|
||||
|
||||
/// Localisation key for the one-line subtitle shown under the label.
|
||||
public var blurbKey: String {
|
||||
switch self {
|
||||
case .speechAnalyzer: return "asr.backend.speechAnalyzer.blurb"
|
||||
case .qwen3ASR: return "asr.backend.qwen3.blurb"
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this backend needs the user to download a model file
|
||||
/// before it can run. Used to gate the "Downloading Qwen3-ASR" UI
|
||||
/// in a follow-up; for now we just expose the flag.
|
||||
public var requiresModelDownload: Bool {
|
||||
switch self {
|
||||
case .speechAnalyzer: return false
|
||||
case .qwen3ASR: return true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// OnDeviceModel.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Identity of an on-device model the host app downloads and the
|
||||
// keyboard extension observes via App Group flags (the extension
|
||||
// cannot read the main app's Caches directory).
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum OnDeviceModel: String, CaseIterable, Identifiable, Sendable {
|
||||
case qwen3ASR
|
||||
|
||||
public var id: String { rawValue }
|
||||
|
||||
/// CoreML inference bundle (`aufklarer/Qwen3-ASR-CoreML`), derived from
|
||||
/// official `Qwen/Qwen3-ASR-0.6B`.
|
||||
public var repoId: String {
|
||||
switch self {
|
||||
case .qwen3ASR: return "aufklarer/Qwen3-ASR-CoreML"
|
||||
}
|
||||
}
|
||||
|
||||
/// Tokenizer files (vocab / merges) pulled from the upstream Qwen repo.
|
||||
public var tokenizerRepoId: String {
|
||||
switch self {
|
||||
case .qwen3ASR: return "Qwen/Qwen3-ASR-0.6B"
|
||||
}
|
||||
}
|
||||
|
||||
public var displayName: String {
|
||||
switch self {
|
||||
case .qwen3ASR: return "Qwen3-ASR 0.6B (CoreML)"
|
||||
}
|
||||
}
|
||||
|
||||
public var approximateSizeMB: Int {
|
||||
switch self {
|
||||
case .qwen3ASR: return 1_600
|
||||
}
|
||||
}
|
||||
|
||||
public var compactSizeLabel: String {
|
||||
"\(approximateSizeMB)M"
|
||||
}
|
||||
|
||||
/// Settings list title: model name plus compact size.
|
||||
public var listTitle: String {
|
||||
"\(displayName) · \(compactSizeLabel)"
|
||||
}
|
||||
|
||||
public var repoAndSizeLabel: String {
|
||||
"\(repoId) · \(approximateSizeMB) MB"
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,11 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
|
||||
static let hasCompletedOnboarding = "config.hasCompletedOnboarding"
|
||||
static let onboardingPage = "config.onboardingPage"
|
||||
static let hasAcknowledgedCloudSharing = "config.hasAcknowledgedCloudSharing"
|
||||
// Which on-device ASR engine to use when `engineMode == "local"`.
|
||||
// Persisted in the App Group so the keyboard can read the
|
||||
// selection even though it never instantiates the backend itself.
|
||||
static let localASRBackend = "config.localASRBackend"
|
||||
static let uiLanguage = "config.uiLanguage"
|
||||
}
|
||||
|
||||
@Published public var providerId: String {
|
||||
@@ -64,8 +69,8 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
|
||||
@Published public var localeId: String {
|
||||
didSet { defaults.set(localeId, forKey: Key.localeId) }
|
||||
}
|
||||
/// "local" → on-device ASR only, no LLM polishing.
|
||||
/// "cloud" → ASR + LLM polish (default).
|
||||
/// "local" → on-device ASR only (raw transcript delivery).
|
||||
/// "cloud" → ASR + LLM polish (always on; modeId kept for compatibility).
|
||||
@Published public var engineMode: String {
|
||||
didSet { defaults.set(engineMode, forKey: Key.engineMode) }
|
||||
}
|
||||
@@ -85,6 +90,16 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
|
||||
@Published public var hasAcknowledgedCloudSharing: Bool {
|
||||
didSet { defaults.set(hasAcknowledgedCloudSharing, forKey: Key.hasAcknowledgedCloudSharing) }
|
||||
}
|
||||
/// Which on-device ASR engine backs the "local" engine mode. Only
|
||||
/// consulted when `isLocalEngine == true`; the cloud engine always
|
||||
/// uses `SpeechAnalyzer`.
|
||||
@Published public var localASRBackend: LocalASRBackend {
|
||||
didSet { defaults.set(localASRBackend.rawValue, forKey: Key.localASRBackend) }
|
||||
}
|
||||
/// Host-app UI language. Also mirrored to the App Group for the keyboard extension.
|
||||
@Published public var uiLanguage: AppUILanguage {
|
||||
didSet { defaults.set(uiLanguage.rawValue, forKey: Key.uiLanguage) }
|
||||
}
|
||||
|
||||
public var isConfigured: Bool {
|
||||
// Local engine (on-device ASR only) doesn't need an API key,
|
||||
@@ -96,7 +111,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
|
||||
return !baseURL.isEmpty && !apiKey.isEmpty && !model.isEmpty
|
||||
}
|
||||
|
||||
/// On-device ASR only — no cloud LLM polish.
|
||||
/// On-device ASR only; no cloud API required.
|
||||
public var isLocalEngine: Bool { engineMode == "local" }
|
||||
|
||||
/// The system prompt the user *sees* in the editor — fall back to the
|
||||
@@ -131,6 +146,19 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
|
||||
let savedPage = resolvedDefaults.integer(forKey: Key.onboardingPage)
|
||||
self.onboardingPage = savedPage > 0 ? savedPage : 0
|
||||
self.hasAcknowledgedCloudSharing = resolvedDefaults.bool(forKey: Key.hasAcknowledgedCloudSharing)
|
||||
// Tolerate missing / unknown raw values (e.g. an enum case that
|
||||
// was renamed in a later build) by falling back to the default
|
||||
// rather than crashing inside `RawRepresentable.init`.
|
||||
let rawBackend = resolvedDefaults.string(forKey: Key.localASRBackend) ?? LocalASRBackend.speechAnalyzer.rawValue
|
||||
self.localASRBackend = LocalASRBackend(rawValue: rawBackend) ?? .speechAnalyzer
|
||||
self.uiLanguage = AppUILanguage.fromStored(
|
||||
resolvedDefaults.string(forKey: Key.uiLanguage)
|
||||
)
|
||||
|
||||
// Cloud no longer exposes off/transcribe; migrate legacy values.
|
||||
if self.engineMode == "cloud", self.modeId != "polish" {
|
||||
self.modeId = "polish"
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the API key from the Keychain, falling back to a one-time
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
// TranscriptionDelivery.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Host-app → keyboard handoff payload: final text plus an optional soft
|
||||
// warning when cloud polish failed but the raw transcript is still delivered.
|
||||
|
||||
import Foundation
|
||||
|
||||
public struct TranscriptionDelivery: Sendable, Equatable {
|
||||
public let text: String
|
||||
public let polishWarning: String?
|
||||
|
||||
public init(text: String, polishWarning: String? = nil) {
|
||||
self.text = text
|
||||
self.polishWarning = polishWarning
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
import Foundation
|
||||
import AVFoundation
|
||||
import CoreMedia
|
||||
import Speech
|
||||
import os
|
||||
|
||||
@@ -42,6 +43,63 @@ public protocol ASRService: Sendable {
|
||||
|
||||
/// Cancel any in-flight recognition and tear down its tasks.
|
||||
func cancel()
|
||||
|
||||
/// Clears cancellation / cached session state before a new utterance.
|
||||
func resetForNewUtterance()
|
||||
|
||||
/// Transcribe one PCM chunk (Flow pipelined path). Default wraps `transcribe(stream:)`.
|
||||
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult
|
||||
}
|
||||
|
||||
public enum ASRChunkResult: Sendable, Equatable {
|
||||
case success(String)
|
||||
case failure(String)
|
||||
case cancelled
|
||||
}
|
||||
|
||||
extension ASRService {
|
||||
public func resetForNewUtterance() {}
|
||||
|
||||
public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
|
||||
guard !samples.isEmpty else { return .success("") }
|
||||
if Task.isCancelled { return .cancelled }
|
||||
|
||||
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
|
||||
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
|
||||
continuation.yield(snapshot)
|
||||
continuation.finish()
|
||||
|
||||
var lastPartial = ""
|
||||
var finalText = ""
|
||||
var failure: String?
|
||||
|
||||
for await event in transcribe(stream: stream, locale: locale) {
|
||||
if Task.isCancelled { return .cancelled }
|
||||
switch event {
|
||||
case .capability:
|
||||
break
|
||||
case .partial(let text):
|
||||
lastPartial = text
|
||||
case .final(let text):
|
||||
finalText = text
|
||||
case .error(let message):
|
||||
failure = message
|
||||
}
|
||||
}
|
||||
|
||||
if let failure {
|
||||
return .failure(failure)
|
||||
}
|
||||
let trimmed = finalText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !trimmed.isEmpty {
|
||||
return .success(trimmed)
|
||||
}
|
||||
let partial = lastPartial.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !partial.isEmpty {
|
||||
return .success(partial)
|
||||
}
|
||||
return .success("")
|
||||
}
|
||||
}
|
||||
|
||||
public enum ASREvent: Sendable, Equatable {
|
||||
@@ -58,13 +116,68 @@ public enum ASREvent: Sendable, Equatable {
|
||||
// MARK: - Factory
|
||||
|
||||
public enum ASRServiceFactory {
|
||||
/// Returns the ASR backend. With iOS 26 as the deployment target,
|
||||
/// there is exactly one backend (`SpeechAnalyzer`).
|
||||
/// Registry of backend-specific providers. The host app installs
|
||||
/// a provider for the Qwen3-ASR backend at launch time (the
|
||||
/// `Qwen3ASRProvider` lives in the app target because linking
|
||||
/// `Qwen3ASR` pulls in mlx-swift, which the shared framework
|
||||
/// deliberately stays off to keep `APPLICATION_EXTENSION_API_ONLY`
|
||||
/// clean). The shared framework always provides a built-in
|
||||
/// `SpeechAnalyzer` provider; custom providers override it.
|
||||
///
|
||||
/// `nonisolated(unsafe)` because the only writer is
|
||||
/// `OSGKeyboardApp.init` (single-threaded, runs once at launch).
|
||||
/// After launch, all callers read the dictionary from any
|
||||
/// actor.
|
||||
public nonisolated(unsafe) static var providers: [LocalASRBackend: any ASRServiceProvider] = [
|
||||
.speechAnalyzer: SpeechAnalyzerProvider()
|
||||
]
|
||||
|
||||
/// Returns the ASR backend chosen by the user. The cloud engine
|
||||
/// always uses the iOS `SpeechAnalyzer` path — it has the lowest
|
||||
/// latency and never hits the network, which matches the user's
|
||||
/// expectation that "ASR" is the local half of the pipeline
|
||||
/// regardless of where the LLM polish happens.
|
||||
///
|
||||
/// For the local engine, we honour `LocalASRBackend`:
|
||||
/// - `.speechAnalyzer` (default) → on-device iOS pipeline.
|
||||
/// - `.qwen3ASR` → CoreML-backed Qwen3-ASR via `soniqo/speech-swift` (host app only)
|
||||
/// (registered by the host app at launch).
|
||||
public static func make(
|
||||
engineMode: String,
|
||||
localBackend: LocalASRBackend = .speechAnalyzer
|
||||
) -> ASRService {
|
||||
if engineMode == "local" {
|
||||
if let provider = providers[localBackend] {
|
||||
return provider.make()
|
||||
}
|
||||
}
|
||||
return SpeechAnalyzerASR()
|
||||
}
|
||||
|
||||
/// Back-compat overload for callers that only ever want the
|
||||
/// SpeechAnalyzer path. The previous single-backend build used
|
||||
/// this signature; new code should pass the engine mode explicitly
|
||||
/// so the user's selection is honoured.
|
||||
public static func make() -> ASRService {
|
||||
SpeechAnalyzerASR()
|
||||
}
|
||||
}
|
||||
|
||||
/// Backend-specific ASR factory. The shared framework ships a default
|
||||
/// `SpeechAnalyzerProvider`; the host app installs a `Qwen3ASRProvider`
|
||||
/// at launch time so the Qwen3 backend is wired in only where its
|
||||
/// large MLX dependency is also linked.
|
||||
public protocol ASRServiceProvider: Sendable {
|
||||
var backend: LocalASRBackend { get }
|
||||
func make() -> ASRService
|
||||
}
|
||||
|
||||
/// Built-in provider for the iOS SpeechAnalyzer path. Always present.
|
||||
struct SpeechAnalyzerProvider: ASRServiceProvider {
|
||||
let backend: LocalASRBackend = .speechAnalyzer
|
||||
func make() -> ASRService { SpeechAnalyzerASR() }
|
||||
}
|
||||
|
||||
// MARK: - PCM format conversion (testable helpers)
|
||||
//
|
||||
// Extracted from the audio-thread hot path so the scaling + clipping
|
||||
@@ -115,6 +228,114 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
private var analyzer: SpeechAnalyzer?
|
||||
private var analyzerTask: Task<Void, Never>?
|
||||
private var analyzerFinished = false
|
||||
/// Reused across pipelined chunks within one utterance (assets + format).
|
||||
private var chunkPreparedLocaleID: String?
|
||||
private var chunkAnalyzerFormat: AVAudioFormat?
|
||||
|
||||
func resetForNewUtterance() {
|
||||
lock.withLock {
|
||||
chunkPreparedLocaleID = nil
|
||||
chunkAnalyzerFormat = nil
|
||||
}
|
||||
}
|
||||
|
||||
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
|
||||
guard !samples.isEmpty else { return .success("") }
|
||||
if Task.isCancelled { return .cancelled }
|
||||
|
||||
do {
|
||||
let text = try await transcribeSamples(samples, locale: locale, reuseChunkPrep: true)
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return trimmed.isEmpty ? .success("") : .success(trimmed)
|
||||
} catch is CancellationError {
|
||||
return .cancelled
|
||||
} catch {
|
||||
return .failure(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
/// Analyze a single PCM buffer without the streaming `transcribe` wrapper.
|
||||
private func transcribeSamples(
|
||||
_ samples: [Float],
|
||||
locale: Locale,
|
||||
reuseChunkPrep: Bool
|
||||
) async throws -> String {
|
||||
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
|
||||
throw ASRChunkError.localeUnsupported
|
||||
}
|
||||
let localeID = resolvedLocale.identifier(.bcp47)
|
||||
let transcriber = DictationTranscriber(
|
||||
locale: resolvedLocale,
|
||||
preset: .progressiveLongDictation
|
||||
)
|
||||
|
||||
let analyzerFormat: AVAudioFormat
|
||||
let cachedPrep = lock.withLock { (chunkPreparedLocaleID, chunkAnalyzerFormat) }
|
||||
if reuseChunkPrep,
|
||||
cachedPrep.0 == localeID,
|
||||
let cached = cachedPrep.1 {
|
||||
analyzerFormat = cached
|
||||
} else {
|
||||
try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
|
||||
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
|
||||
compatibleWith: [transcriber],
|
||||
considering: Self.captureFormat
|
||||
) else {
|
||||
throw ASRChunkError.formatUnsupported
|
||||
}
|
||||
analyzerFormat = format
|
||||
lock.withLock {
|
||||
chunkPreparedLocaleID = localeID
|
||||
chunkAnalyzerFormat = format
|
||||
}
|
||||
}
|
||||
|
||||
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
|
||||
guard let pcm = Self.makeAnalyzerPCMBuffer(from: snapshot, format: analyzerFormat) else {
|
||||
throw ASRChunkError.formatUnsupported
|
||||
}
|
||||
|
||||
let analyzer = SpeechAnalyzer(modules: [transcriber])
|
||||
try await analyzer.prepareToAnalyze(in: analyzerFormat)
|
||||
|
||||
let resultsTask = Task<String, Error> {
|
||||
var accumulator = ProgressiveDictationTranscriptAccumulator()
|
||||
for try await result in transcriber.results {
|
||||
if Task.isCancelled { break }
|
||||
let text = String(result.text.characters)
|
||||
_ = accumulator.ingest(range: result.range, text: text)
|
||||
}
|
||||
return accumulator.finalize()
|
||||
}
|
||||
|
||||
let inputStream = AsyncStream<AnalyzerInput> { continuation in
|
||||
continuation.yield(AnalyzerInput(buffer: pcm))
|
||||
continuation.finish()
|
||||
}
|
||||
|
||||
let lastSampleTime = try await analyzer.analyzeSequence(inputStream)
|
||||
if let lastSampleTime {
|
||||
try await analyzer.finalizeAndFinish(through: lastSampleTime)
|
||||
} else {
|
||||
await analyzer.cancelAndFinishNow()
|
||||
}
|
||||
|
||||
return try await resultsTask.value
|
||||
}
|
||||
|
||||
private enum ASRChunkError: LocalizedError {
|
||||
case localeUnsupported
|
||||
case formatUnsupported
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .localeUnsupported:
|
||||
return SharedL10n.string("error.asr.localeUnsupported")
|
||||
case .formatUnsupported:
|
||||
return SharedL10n.string("error.asr.formatUnsupported")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical capture format: 16 kHz mono Float32 from `AudioCaptureService`
|
||||
/// / `PreviewASRController` before it reaches SpeechAnalyzer.
|
||||
@@ -146,16 +367,21 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
do {
|
||||
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
|
||||
Self.debug("locale unsupported: \(locale.identifier(.bcp47))")
|
||||
continuation.yield(.error("当前系统未分配可用语音语言模型,请稍后重试或切换语言"))
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.localeUnsupported")))
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
let transcriber = DictationTranscriber(locale: resolvedLocale, preset: .progressiveShortDictation)
|
||||
// Each pipelined chunk is ≤ 30 s; long dictation preset keeps a
|
||||
// single chunk coherent (Flow utterances run up to 3 min).
|
||||
let transcriber = DictationTranscriber(
|
||||
locale: resolvedLocale,
|
||||
preset: .progressiveLongDictation
|
||||
)
|
||||
do {
|
||||
try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
|
||||
} catch {
|
||||
Self.debug("asset prepare failed: \(error.localizedDescription)")
|
||||
continuation.yield(.error("语音语言资源未就绪,请稍后重试"))
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.assetsNotReady")))
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
@@ -167,7 +393,7 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
compatibleWith: [transcriber],
|
||||
considering: Self.captureFormat
|
||||
) else {
|
||||
continuation.yield(.error("当前设备不支持该语音输入格式"))
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.formatUnsupported")))
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
@@ -179,15 +405,16 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
// Apple recommends consuming `transcriber.results` concurrently
|
||||
// while `analyzeSequence` drains the input stream.
|
||||
let resultsTask = Task<String, Error> {
|
||||
var lastText = ""
|
||||
var accumulator = ProgressiveDictationTranscriptAccumulator()
|
||||
for try await result in transcriber.results {
|
||||
if Task.isCancelled { break }
|
||||
let text = String(result.text.characters)
|
||||
guard !text.isEmpty, text != lastText else { continue }
|
||||
lastText = text
|
||||
continuation.yield(.partial(text))
|
||||
guard let full = accumulator.ingest(range: result.range, text: text) else {
|
||||
continue
|
||||
}
|
||||
continuation.yield(.partial(full))
|
||||
}
|
||||
return lastText
|
||||
return accumulator.finalize()
|
||||
}
|
||||
|
||||
let lastSampleTime = try await newAnalyzer.analyzeSequence(inputStream)
|
||||
@@ -195,7 +422,7 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
if let lastSampleTime {
|
||||
try await newAnalyzer.finalizeAndFinish(through: lastSampleTime)
|
||||
} else {
|
||||
try await newAnalyzer.cancelAndFinishNow()
|
||||
await newAnalyzer.cancelAndFinishNow()
|
||||
}
|
||||
|
||||
let lastText: String
|
||||
@@ -210,7 +437,7 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
|
||||
let trimmed = lastText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if trimmed.isEmpty {
|
||||
continuation.yield(.error("未识别到语音内容,请重试"))
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
|
||||
} else {
|
||||
continuation.yield(.final(trimmed))
|
||||
}
|
||||
|
||||
@@ -35,6 +35,8 @@ public struct AppGroupStore: @unchecked Sendable {
|
||||
static let modeId = "config.modeId"
|
||||
static let localeId = "config.localeId"
|
||||
static let engineMode = "config.engineMode"
|
||||
static let localASRBackend = "config.localASRBackend"
|
||||
static let uiLanguage = "config.uiLanguage"
|
||||
}
|
||||
|
||||
// MARK: - Reads
|
||||
@@ -70,12 +72,25 @@ public struct AppGroupStore: @unchecked Sendable {
|
||||
defaults.string(forKey: Key.localeId) ?? "auto"
|
||||
}
|
||||
|
||||
/// "local" → on-device ASR only, no LLM polishing.
|
||||
/// "local" → on-device ASR only (raw transcript delivery).
|
||||
/// "cloud" → ASR + LLM polish (default behaviour).
|
||||
public var engineMode: String {
|
||||
defaults.string(forKey: Key.engineMode) ?? "cloud"
|
||||
}
|
||||
|
||||
/// Which on-device ASR engine backs the "local" engine mode. Falls
|
||||
/// back to the iOS SpeechAnalyzer path so legacy installs (which
|
||||
/// never wrote this key) keep working.
|
||||
public var localASRBackend: LocalASRBackend {
|
||||
let raw = defaults.string(forKey: Key.localASRBackend) ?? LocalASRBackend.speechAnalyzer.rawValue
|
||||
return LocalASRBackend(rawValue: raw) ?? .speechAnalyzer
|
||||
}
|
||||
|
||||
/// Host-app UI language override (`auto` / `en` / `zh-Hans`).
|
||||
public var uiLanguage: AppUILanguage {
|
||||
AppUILanguage.fromStored(defaults.string(forKey: Key.uiLanguage))
|
||||
}
|
||||
|
||||
// MARK: - Writes
|
||||
|
||||
public func setModeId(_ id: String) {
|
||||
@@ -90,6 +105,14 @@ public struct AppGroupStore: @unchecked Sendable {
|
||||
defaults.set(mode, forKey: Key.engineMode)
|
||||
}
|
||||
|
||||
public func setLocalASRBackend(_ backend: LocalASRBackend) {
|
||||
defaults.set(backend.rawValue, forKey: Key.localASRBackend)
|
||||
}
|
||||
|
||||
public func setUILanguage(_ language: AppUILanguage) {
|
||||
defaults.set(language.rawValue, forKey: Key.uiLanguage)
|
||||
}
|
||||
|
||||
// MARK: - Client
|
||||
|
||||
public func makeClient() -> LLMClient {
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
// ChunkedUtterancePipeline.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Pipelined Flow utterance ASR: split PCM while recording, transcribe chunks
|
||||
// serially on a background queue, stitch partials for display and delivery.
|
||||
|
||||
import Foundation
|
||||
|
||||
public struct ChunkedUtteranceSuccess: Sendable, Equatable {
|
||||
public let text: String
|
||||
/// Non-fatal per-chunk ASR issues (delivered as soft warning when non-empty).
|
||||
public let chunkWarnings: [String]
|
||||
|
||||
public init(text: String, chunkWarnings: [String] = []) {
|
||||
self.text = text
|
||||
self.chunkWarnings = chunkWarnings
|
||||
}
|
||||
}
|
||||
|
||||
public enum ChunkedUtterancePipelineOutcome: Sendable, Equatable {
|
||||
case success(ChunkedUtteranceSuccess)
|
||||
case failure(String)
|
||||
case cancelled
|
||||
}
|
||||
|
||||
/// Thread-safe queue between the chunk feeder and ASR worker.
|
||||
private actor ChunkWorkQueue {
|
||||
private var items: [UtteranceAudioChunk] = []
|
||||
private var finished = false
|
||||
private var waiters: [CheckedContinuation<UtteranceAudioChunk?, Never>] = []
|
||||
|
||||
func enqueue(_ chunk: UtteranceAudioChunk) {
|
||||
items.append(chunk)
|
||||
resumeWaiters()
|
||||
}
|
||||
|
||||
func markFinished() {
|
||||
finished = true
|
||||
resumeWaiters()
|
||||
}
|
||||
|
||||
func dequeue() async -> UtteranceAudioChunk? {
|
||||
if !items.isEmpty {
|
||||
return items.removeFirst()
|
||||
}
|
||||
if finished {
|
||||
return nil
|
||||
}
|
||||
return await withCheckedContinuation { continuation in
|
||||
waiters.append(continuation)
|
||||
}
|
||||
}
|
||||
|
||||
private func resumeWaiters() {
|
||||
while !waiters.isEmpty {
|
||||
if !items.isEmpty {
|
||||
let waiter = waiters.removeFirst()
|
||||
waiter.resume(returning: items.removeFirst())
|
||||
} else if finished {
|
||||
let waiter = waiters.removeFirst()
|
||||
waiter.resume(returning: nil)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public actor ChunkedUtterancePipeline {
|
||||
private let asr: ASRService
|
||||
private let locale: Locale
|
||||
private let config: FlowUtteranceChunkConfig
|
||||
private var cancelled = false
|
||||
|
||||
public init(
|
||||
asr: ASRService,
|
||||
locale: Locale,
|
||||
config: FlowUtteranceChunkConfig = .flowDefault
|
||||
) {
|
||||
self.asr = asr
|
||||
self.locale = locale
|
||||
self.config = config
|
||||
}
|
||||
|
||||
public func cancel() {
|
||||
cancelled = true
|
||||
asr.cancel()
|
||||
}
|
||||
|
||||
/// Consume `stream` until finished; ASR runs off the caller's actor while recording continues.
|
||||
public func transcribe(
|
||||
stream: AsyncStream<AudioBufferSnapshot>,
|
||||
onPartial: @Sendable @escaping (String) -> Void
|
||||
) async -> ChunkedUtterancePipelineOutcome {
|
||||
asr.resetForNewUtterance()
|
||||
|
||||
let queue = ChunkWorkQueue()
|
||||
var stitcher = UtteranceTranscriptStitcher()
|
||||
var chunkWarnings: [String] = []
|
||||
var failedChunks = 0
|
||||
var processedChunks = 0
|
||||
|
||||
let feeder = Task {
|
||||
for await chunk in UtteranceStreamChunker.chunks(from: stream, config: config) {
|
||||
if Task.isCancelled { break }
|
||||
await queue.enqueue(chunk)
|
||||
}
|
||||
await queue.markFinished()
|
||||
}
|
||||
|
||||
while true {
|
||||
if cancelled || Task.isCancelled {
|
||||
feeder.cancel()
|
||||
return .cancelled
|
||||
}
|
||||
|
||||
guard let chunk = await queue.dequeue() else { break }
|
||||
|
||||
processedChunks += 1
|
||||
let asr = self.asr
|
||||
let locale = self.locale
|
||||
let result = await Task.detached(priority: .userInitiated) {
|
||||
await asr.transcribeChunk(samples: chunk.samples, locale: locale)
|
||||
}.value
|
||||
|
||||
switch result {
|
||||
case .success(let text):
|
||||
stitcher.append(index: chunk.index, text: text)
|
||||
let partial = stitcher.composed()
|
||||
if !partial.isEmpty {
|
||||
onPartial(partial)
|
||||
}
|
||||
case .failure(let message):
|
||||
failedChunks += 1
|
||||
chunkWarnings.append(
|
||||
SharedL10n.format(
|
||||
"error.asr.chunkFailed",
|
||||
chunk.index + 1,
|
||||
message
|
||||
)
|
||||
)
|
||||
case .cancelled:
|
||||
feeder.cancel()
|
||||
return .cancelled
|
||||
}
|
||||
}
|
||||
|
||||
_ = await feeder.value
|
||||
|
||||
let finalText = stitcher.composed().trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if finalText.isEmpty {
|
||||
if failedChunks > 0, processedChunks == failedChunks {
|
||||
return .failure(SharedL10n.string("error.asr.noSpeech"))
|
||||
}
|
||||
return .failure(SharedL10n.string("error.asr.noSpeech"))
|
||||
}
|
||||
|
||||
return .success(ChunkedUtteranceSuccess(text: finalText, chunkWarnings: chunkWarnings))
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ public enum DictationBridge {
|
||||
|
||||
private enum Key {
|
||||
static let pendingText = "dictation.pendingText"
|
||||
static let polishWarning = "dictation.polishWarning"
|
||||
static let updatedAt = "dictation.updatedAt"
|
||||
static let status = "dictation.status"
|
||||
static let statusUpdatedAt = "dictation.statusUpdatedAt"
|
||||
@@ -67,12 +68,21 @@ public enum DictationBridge {
|
||||
}
|
||||
|
||||
/// Store a transcript for the keyboard extension to consume.
|
||||
public static func storePendingTranscript(_ text: String, defaults: UserDefaults? = nil) {
|
||||
public static func storePendingTranscript(
|
||||
_ text: String,
|
||||
polishWarning: String? = nil,
|
||||
defaults: UserDefaults? = nil
|
||||
) {
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return }
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.set(trimmed, forKey: Key.pendingText)
|
||||
store.set(Date().timeIntervalSince1970, forKey: Key.updatedAt)
|
||||
if let polishWarning, !polishWarning.isEmpty {
|
||||
store.set(polishWarning, forKey: Key.polishWarning)
|
||||
} else {
|
||||
store.removeObject(forKey: Key.polishWarning)
|
||||
}
|
||||
setStatus(.done, defaults: store)
|
||||
}
|
||||
|
||||
@@ -81,6 +91,15 @@ public enum DictationBridge {
|
||||
maxAge: TimeInterval = 180,
|
||||
defaults: UserDefaults? = nil
|
||||
) -> String? {
|
||||
consumePendingDelivery(maxAge: maxAge, defaults: defaults)?.text
|
||||
}
|
||||
|
||||
/// Returns and clears the pending delivery (text + optional polish
|
||||
/// warning) if present.
|
||||
public static func consumePendingDelivery(
|
||||
maxAge: TimeInterval = 180,
|
||||
defaults: UserDefaults? = nil
|
||||
) -> TranscriptionDelivery? {
|
||||
let store = resolvedDefaults(defaults)
|
||||
guard let text = store.string(forKey: Key.pendingText) else {
|
||||
return nil
|
||||
@@ -92,14 +111,18 @@ public enum DictationBridge {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
let warning = store.string(forKey: Key.polishWarning)
|
||||
store.removeObject(forKey: Key.pendingText)
|
||||
store.removeObject(forKey: Key.polishWarning)
|
||||
store.removeObject(forKey: Key.updatedAt)
|
||||
setStatus(.idle, defaults: store)
|
||||
return text
|
||||
return TranscriptionDelivery(text: text, polishWarning: warning)
|
||||
}
|
||||
|
||||
public static func clear(defaults: UserDefaults? = nil) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.removeObject(forKey: Key.pendingText)
|
||||
store.removeObject(forKey: Key.polishWarning)
|
||||
store.removeObject(forKey: Key.updatedAt)
|
||||
setStatus(.idle, defaults: store)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
// FlowAppLifecycle.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Tracks whether the host app process is in the foreground.
|
||||
// Retained for any future GPU-backed paths; CoreML ASR does not require it.
|
||||
|
||||
import Foundation
|
||||
|
||||
public final class FlowAppLifecycle: @unchecked Sendable {
|
||||
|
||||
public static let shared = FlowAppLifecycle()
|
||||
|
||||
private let lock = NSLock()
|
||||
private var isForeground = true
|
||||
|
||||
private init() {}
|
||||
|
||||
/// `true` when the host app scene is active (`.active`).
|
||||
public var allowsGPUInference: Bool {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return isForeground
|
||||
}
|
||||
|
||||
public func setForeground(_ foreground: Bool) {
|
||||
lock.lock()
|
||||
isForeground = foreground
|
||||
lock.unlock()
|
||||
}
|
||||
|
||||
/// Blocks until foreground or cancellation.
|
||||
public func waitUntilForeground() async -> Bool {
|
||||
while !allowsGPUInference {
|
||||
if Task.isCancelled { return false }
|
||||
try? await Task.sleep(nanoseconds: 200_000_000)
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,7 @@ private final class FlowCaptureStreamRelay: @unchecked Sendable {
|
||||
}
|
||||
|
||||
func yield(_ snapshot: AudioBufferSnapshot) {
|
||||
lock.withLock { continuation?.yield(snapshot) }
|
||||
_ = lock.withLock { continuation?.yield(snapshot) }
|
||||
}
|
||||
|
||||
func finish() {
|
||||
@@ -250,6 +250,22 @@ public final class FlowContinuousCapture {
|
||||
)
|
||||
}
|
||||
|
||||
/// Re-activate capture after returning from background without
|
||||
/// reinstalling the tap (iOS may deactivate the audio session).
|
||||
public func reassertIfRunning() {
|
||||
guard isRunning else { return }
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
try? session.setCategory(
|
||||
.playAndRecord,
|
||||
mode: .measurement,
|
||||
options: [.defaultToSpeaker, .allowBluetoothHFP, .mixWithOthers]
|
||||
)
|
||||
try? session.setActive(true, options: .notifyOthersOnDeactivation)
|
||||
if !audioEngine.isRunning {
|
||||
try? audioEngine.start()
|
||||
}
|
||||
}
|
||||
|
||||
/// Begin forwarding downsampled buffers to ASR for one utterance.
|
||||
public func beginUtterance() -> AsyncStream<AudioBufferSnapshot> {
|
||||
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
|
||||
|
||||
@@ -65,12 +65,24 @@ public enum FlowSessionBridge {
|
||||
|
||||
// MARK: - Session validity (keyboard)
|
||||
|
||||
/// True when expires is in the future and heartbeat is fresh.
|
||||
/// True when the session contract is still valid (not expired).
|
||||
/// Does not require a fresh heartbeat — the host may be suspended in
|
||||
/// background while the continuous audio session is frozen.
|
||||
public static func isSessionActive(defaults: UserDefaults? = nil) -> Bool {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
guard store.bool(forKey: FlowSessionKeys.flowSessionActive) else { return false }
|
||||
|
||||
let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires)
|
||||
guard expires > Date().timeIntervalSince1970 else { return false }
|
||||
return expires > Date().timeIntervalSince1970
|
||||
}
|
||||
|
||||
/// True when the host app recently wrote a heartbeat (foreground or
|
||||
/// actively processing). Used for auto-start heuristics, not gating record.
|
||||
public static func isHostReachable(defaults: UserDefaults? = nil) -> Bool {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
guard isSessionActive(defaults: store) else { return false }
|
||||
|
||||
let heartbeat = store.double(forKey: FlowSessionKeys.flowHeartbeat)
|
||||
guard heartbeat > 0 else { return false }
|
||||
@@ -124,6 +136,7 @@ public enum FlowSessionBridge {
|
||||
|
||||
public static func storeTranscriptionResult(
|
||||
_ text: String,
|
||||
polishWarning: String? = nil,
|
||||
defaults: UserDefaults? = nil
|
||||
) {
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
@@ -131,6 +144,11 @@ public enum FlowSessionBridge {
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.set(trimmed, forKey: FlowSessionKeys.transcriptionResult)
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionError)
|
||||
if let polishWarning, !polishWarning.isEmpty {
|
||||
store.set(polishWarning, forKey: FlowSessionKeys.transcriptionPolishWarning)
|
||||
} else {
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionPolishWarning)
|
||||
}
|
||||
setRecordingState(.idle, defaults: store)
|
||||
flush(store)
|
||||
}
|
||||
@@ -147,14 +165,24 @@ public enum FlowSessionBridge {
|
||||
|
||||
/// Returns and clears a pending transcription result, if any.
|
||||
public static func consumeTranscriptionResult(defaults: UserDefaults? = nil) -> String? {
|
||||
consumeTranscriptionDelivery(defaults: defaults)?.text
|
||||
}
|
||||
|
||||
/// Returns and clears a pending transcription delivery (text + optional
|
||||
/// polish warning), if any.
|
||||
public static func consumeTranscriptionDelivery(
|
||||
defaults: UserDefaults? = nil
|
||||
) -> TranscriptionDelivery? {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
guard let text = store.string(forKey: FlowSessionKeys.transcriptionResult), !text.isEmpty else {
|
||||
return nil
|
||||
}
|
||||
let warning = store.string(forKey: FlowSessionKeys.transcriptionPolishWarning)
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionResult)
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionPolishWarning)
|
||||
flush(store)
|
||||
return text
|
||||
return TranscriptionDelivery(text: text, polishWarning: warning)
|
||||
}
|
||||
|
||||
/// Returns and clears a pending transcription error, if any.
|
||||
@@ -212,6 +240,7 @@ public enum FlowSessionBridge {
|
||||
|
||||
private static func clearTranscription(defaults: UserDefaults) {
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionResult)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionPolishWarning)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionError)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,17 +13,39 @@ public enum FlowSessionKeys {
|
||||
public static let keyboardRecordingState = "flow.keyboardRecordingState"
|
||||
public static let transcriptionLanguage = "flow.transcriptionLanguage"
|
||||
public static let transcriptionResult = "flow.transcriptionResult"
|
||||
/// Soft warning when polish failed but raw transcript was delivered.
|
||||
public static let transcriptionPolishWarning = "flow.transcriptionPolishWarning"
|
||||
public static let transcriptionError = "flow.transcriptionError"
|
||||
public static let audioLevels = "flow.audioLevels"
|
||||
|
||||
/// Heartbeat older than this implies the host app was killed.
|
||||
/// Heartbeat older than this while the host is foreground → likely killed.
|
||||
public static let heartbeatStaleInterval: TimeInterval = 3
|
||||
|
||||
/// Default Flow session length when started from the keyboard.
|
||||
public static let defaultSessionDuration: TimeInterval = 480
|
||||
|
||||
/// Maximum duration for a single keyboard utterance.
|
||||
public static let maxUtteranceDuration: TimeInterval = 60
|
||||
/// Maximum duration for a single keyboard utterance (3 minutes).
|
||||
public static let maxUtteranceDuration: TimeInterval = 180
|
||||
|
||||
/// Host polls for pipelined ASR drain after mic stop. Pipelining usually
|
||||
/// finishes most chunks during recording; this is a soft deadline before
|
||||
/// blocking on `asrTask.value` (which waits until the pipeline exits).
|
||||
public static let localASRWaitTimeout: TimeInterval = 120
|
||||
public static let localQwen3ASRWaitTimeout: TimeInterval = 180
|
||||
public static let cloudASRWaitTimeout: TimeInterval = 120
|
||||
|
||||
/// Keyboard watchdog after the user stops recording (not utterance max length).
|
||||
/// Must cover worst-case post-stop backlog: remaining MLX/SpeechAnalyzer chunks
|
||||
/// plus cloud LLM polish (see `PolishingService.effectiveTimeout` cap).
|
||||
public static func keyboardResultTimeout(
|
||||
engineMode: String,
|
||||
localASRBackend: LocalASRBackend
|
||||
) -> TimeInterval {
|
||||
if engineMode == "local" {
|
||||
return localASRBackend == .qwen3ASR ? 240 : 180
|
||||
}
|
||||
return 240
|
||||
}
|
||||
|
||||
public enum RecordingState: String, Sendable, Equatable {
|
||||
case idle
|
||||
|
||||
@@ -71,8 +71,17 @@ public final class KeyboardState: ObservableObject {
|
||||
@Published public var utteranceRemainingSeconds: Int = Int(FlowSessionKeys.maxUtteranceDuration)
|
||||
/// Whether the host app's Flow voice session is currently valid.
|
||||
@Published public var flowSessionActive: Bool = false
|
||||
/// "local" → ASR only, no LLM. "cloud" → ASR + optional LLM polish.
|
||||
/// "local" → on-device ASR only. "cloud" → ASR + LLM polish.
|
||||
@Published public var engineMode: String = "cloud"
|
||||
/// Which on-device ASR engine to use when `engineMode == "local"`.
|
||||
/// Mirrored from `ProviderConfig.localASRBackend` for UI display
|
||||
/// and for `state` consumers that want a single source of truth.
|
||||
@Published public var localASRBackend: LocalASRBackend = .speechAnalyzer
|
||||
/// `false` when the local engine needs on-device models that are
|
||||
/// not yet downloaded (mirrored from App Group by the extension).
|
||||
@Published public var localModelsReady: Bool = true
|
||||
/// `true` when host app has preloaded Qwen weights into memory.
|
||||
@Published public var localModelsLoaded: Bool = false
|
||||
|
||||
/// Convenience shorthand used by the pipeline and views.
|
||||
public var isLocalEngine: Bool { engineMode == "local" }
|
||||
@@ -86,6 +95,7 @@ public final class KeyboardState: ObservableObject {
|
||||
public var setMode: (InputMode) -> Void = { _ in }
|
||||
public var setLocale: (String) -> Void = { _ in }
|
||||
public var setEngineMode: (String) -> Void = { _ in }
|
||||
public var setLocalASRBackend: (LocalASRBackend) -> Void = { _ in }
|
||||
public var insertNewline: () -> Void = {}
|
||||
public var insertSpace: () -> Void = {}
|
||||
public var deleteBackward: () -> Void = {}
|
||||
|
||||
@@ -17,13 +17,20 @@ public enum LLMError: Error, LocalizedError, Sendable, Equatable {
|
||||
|
||||
public var errorDescription: String? {
|
||||
switch self {
|
||||
case .invalidURL: return "API 地址无效。请在设置中检查 Base URL。"
|
||||
case .noAPIKey: return "未填写 API Key。"
|
||||
case .http(let s): return "API 返回 HTTP \(s)。请稍后重试或联系服务方。"
|
||||
case .decoding: return "解析 API 响应失败。"
|
||||
case .transport: return "网络错误,请检查连接后重试。"
|
||||
case .rateLimited: return "API 调用过于频繁,请稍候再试。"
|
||||
case .cancelled: return "请求已取消。"
|
||||
case .invalidURL:
|
||||
return SharedL10n.string("error.llm.invalidURL")
|
||||
case .noAPIKey:
|
||||
return SharedL10n.string("error.llm.noAPIKey")
|
||||
case .http(let status):
|
||||
return SharedL10n.format("error.llm.http", status)
|
||||
case .decoding:
|
||||
return SharedL10n.string("error.llm.decoding")
|
||||
case .transport:
|
||||
return SharedL10n.string("error.llm.transport")
|
||||
case .rateLimited:
|
||||
return SharedL10n.string("error.llm.rateLimited")
|
||||
case .cancelled:
|
||||
return SharedL10n.string("error.llm.cancelled")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ private final class CaptureStreamRelay: @unchecked Sendable {
|
||||
}
|
||||
|
||||
func yield(_ snapshot: AudioBufferSnapshot) {
|
||||
lock.withLock { continuation?.yield(snapshot) }
|
||||
_ = lock.withLock { continuation?.yield(snapshot) }
|
||||
}
|
||||
|
||||
func finish() {
|
||||
@@ -73,7 +73,7 @@ public final class LiveDictationController: ObservableObject {
|
||||
/// next recording starts from zero.
|
||||
@Published public var lastFinal: String = ""
|
||||
|
||||
private let asr: ASRService = ASRServiceFactory.make()
|
||||
private let asr: ASRService
|
||||
private let audioEngine = AVAudioEngine()
|
||||
/// `internal` (not `private`) so the regression test in
|
||||
/// `OSGKeyboardTests/PreviewASRControllerStateTests.swift` can
|
||||
@@ -83,10 +83,20 @@ public final class LiveDictationController: ObservableObject {
|
||||
/// code outside the class from racing on it.
|
||||
public var asrTask: Task<Void, Never>?
|
||||
private let streamRelay = CaptureStreamRelay()
|
||||
private var chunkedPipeline: ChunkedUtterancePipeline?
|
||||
private var didConfigureAudioSession = false
|
||||
private var didInstallTap = false
|
||||
|
||||
public init() {}
|
||||
public init(asr: ASRService? = nil) {
|
||||
// Resolve through the factory so the user's `LocalASRBackend`
|
||||
// selection is honoured. Tests can pass a stub `asr` directly
|
||||
// to bypass the factory and exercise the controller in
|
||||
// isolation.
|
||||
self.asr = asr ?? ASRServiceFactory.make(
|
||||
engineMode: ProviderConfig.shared.engineMode,
|
||||
localBackend: ProviderConfig.shared.localASRBackend
|
||||
)
|
||||
}
|
||||
|
||||
/// Start dictation using a persisted settings locale id (`auto`, `zh-Hans`, …).
|
||||
public func start(localeId: String) async {
|
||||
@@ -112,6 +122,10 @@ public final class LiveDictationController: ObservableObject {
|
||||
// same `events` stream.
|
||||
asrTask?.cancel()
|
||||
asrTask = nil
|
||||
if let pipeline = chunkedPipeline {
|
||||
Task { await pipeline.cancel() }
|
||||
}
|
||||
chunkedPipeline = nil
|
||||
teardownCapturePipeline()
|
||||
phase = .requestingPermission
|
||||
currentPartial = ""
|
||||
@@ -335,29 +349,36 @@ public final class LiveDictationController: ObservableObject {
|
||||
return
|
||||
}
|
||||
|
||||
// 5. Wire up ASR.
|
||||
let events = asr.transcribe(
|
||||
stream: stream,
|
||||
locale: locale
|
||||
)
|
||||
asrTask = Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
for await event in events {
|
||||
switch event {
|
||||
case .capability:
|
||||
break
|
||||
case .partial(let s):
|
||||
self.currentPartial = s
|
||||
case .final(let s):
|
||||
let trimmed = s.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.lastFinal = trimmed
|
||||
self.currentPartial = ""
|
||||
self.phase = .idle
|
||||
case .error(let m):
|
||||
self.debug("asr error: \(m)")
|
||||
self.teardownCapturePipeline()
|
||||
self.errorMessage = m
|
||||
self.phase = .error(m)
|
||||
// 5. Pipelined ASR (same chunk path as Flow host).
|
||||
let pipeline = ChunkedUtterancePipeline(asr: asr, locale: locale)
|
||||
chunkedPipeline = pipeline
|
||||
asrTask = Task.detached(priority: .userInitiated) { [weak controller = self] in
|
||||
let outcome = await pipeline.transcribe(stream: stream) { partial in
|
||||
Task { @MainActor in
|
||||
controller?.currentPartial = partial
|
||||
}
|
||||
}
|
||||
await MainActor.run {
|
||||
guard let controller else { return }
|
||||
switch outcome {
|
||||
case .success(let success):
|
||||
let trimmed = success.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !trimmed.isEmpty {
|
||||
controller.lastFinal = trimmed
|
||||
controller.currentPartial = ""
|
||||
}
|
||||
if controller.phase == .processing || controller.phase == .recording {
|
||||
controller.phase = .idle
|
||||
}
|
||||
case .failure(let message):
|
||||
controller.debug("asr error: \(message)")
|
||||
controller.teardownCapturePipeline()
|
||||
controller.errorMessage = message
|
||||
controller.phase = .error(message)
|
||||
case .cancelled:
|
||||
if controller.phase == .processing {
|
||||
controller.phase = .idle
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
// OnDeviceModelStatus.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Mirrors on-device model download state into the App Group so the
|
||||
// keyboard extension can show readiness hints without reading the
|
||||
// host app's Caches directory.
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum OnDeviceModelStatus {
|
||||
|
||||
private enum Key {
|
||||
static func downloaded(_ model: OnDeviceModel) -> String {
|
||||
"models.\(model.rawValue).downloaded"
|
||||
}
|
||||
static func progress(_ model: OnDeviceModel) -> String {
|
||||
"models.\(model.rawValue).downloadProgress"
|
||||
}
|
||||
static let modelsLoadedInMemory = "models.loadedInMemory"
|
||||
}
|
||||
|
||||
// MARK: - Writes (host app)
|
||||
|
||||
public static func setDownloaded(_ downloaded: Bool, for model: OnDeviceModel) {
|
||||
guard AppGroup.isAvailable else { return }
|
||||
AppGroup.defaults.set(downloaded, forKey: Key.downloaded(model))
|
||||
if downloaded {
|
||||
clearProgress(for: model)
|
||||
}
|
||||
}
|
||||
|
||||
public static func setProgress(_ progress: Double?, for model: OnDeviceModel) {
|
||||
guard AppGroup.isAvailable else { return }
|
||||
if let progress {
|
||||
AppGroup.defaults.set(progress, forKey: Key.progress(model))
|
||||
} else {
|
||||
AppGroup.defaults.removeObject(forKey: Key.progress(model))
|
||||
}
|
||||
}
|
||||
|
||||
public static func clearProgress(for model: OnDeviceModel) {
|
||||
guard AppGroup.isAvailable else { return }
|
||||
AppGroup.defaults.removeObject(forKey: Key.progress(model))
|
||||
}
|
||||
|
||||
public static func setModelsLoadedInMemory(_ loaded: Bool) {
|
||||
guard AppGroup.isAvailable else { return }
|
||||
AppGroup.defaults.set(loaded, forKey: Key.modelsLoadedInMemory)
|
||||
}
|
||||
|
||||
public static func modelsLoadedInMemory(defaults: UserDefaults? = nil) -> Bool {
|
||||
let store = defaults ?? (AppGroup.isAvailable ? AppGroup.defaults : .standard)
|
||||
return store.bool(forKey: Key.modelsLoadedInMemory)
|
||||
}
|
||||
|
||||
// MARK: - Reads (keyboard + host app)
|
||||
|
||||
public static func isDownloaded(_ model: OnDeviceModel, defaults: UserDefaults? = nil) -> Bool {
|
||||
let store = defaults ?? (AppGroup.isAvailable ? AppGroup.defaults : .standard)
|
||||
return store.bool(forKey: Key.downloaded(model))
|
||||
}
|
||||
|
||||
public static func downloadProgress(_ model: OnDeviceModel, defaults: UserDefaults? = nil) -> Double? {
|
||||
let store = defaults ?? (AppGroup.isAvailable ? AppGroup.defaults : .standard)
|
||||
guard store.object(forKey: Key.progress(model)) != nil else { return nil }
|
||||
return store.double(forKey: Key.progress(model))
|
||||
}
|
||||
|
||||
/// Whether the currently selected local-engine stack has every
|
||||
/// required on-device model downloaded.
|
||||
public static func isLocalStackReady(
|
||||
asrBackend: LocalASRBackend,
|
||||
defaults: UserDefaults? = nil
|
||||
) -> Bool {
|
||||
if asrBackend == .qwen3ASR {
|
||||
return isDownloaded(.qwen3ASR, defaults: defaults)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/// First missing model for the active local stack, if any.
|
||||
public static func firstMissingModel(
|
||||
asrBackend: LocalASRBackend,
|
||||
defaults: UserDefaults? = nil
|
||||
) -> OnDeviceModel? {
|
||||
if asrBackend == .qwen3ASR, !isDownloaded(.qwen3ASR, defaults: defaults) {
|
||||
return .qwen3ASR
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - On-device Qwen3 runtime
|
||||
|
||||
/// CoreML ASR requires iOS 18+ / macOS 15+ (MLState KV cache).
|
||||
public enum OnDeviceMLRuntime {
|
||||
/// Whether Qwen3-ASR CoreML can run in this process.
|
||||
public static var supportsOnDeviceQwen3: Bool {
|
||||
if #available(iOS 18.0, *) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,9 @@
|
||||
// to produce polished, well-punctuated text. Falls back to the raw transcript
|
||||
// if the LLM call fails or times out.
|
||||
//
|
||||
// Mode-aware: when `modeId == "off"` the service short-circuits and returns
|
||||
// the trimmed input without touching the network. This is the runtime
|
||||
// guarantee behind the keyboard's "Off · 关闭" mode.
|
||||
// Cloud engine always runs the LLM polish step (settings no longer expose
|
||||
// off / transcribe). Local engine (`engineMode == "local"`) is ASR-only —
|
||||
// the raw transcript is returned unchanged and cloud API settings are ignored.
|
||||
|
||||
import Foundation
|
||||
|
||||
@@ -16,7 +16,6 @@ public actor PolishingService {
|
||||
public enum PolishError: Error, Equatable {
|
||||
case noTranscript
|
||||
case timeout
|
||||
case modeOff
|
||||
}
|
||||
|
||||
private let store: AppGroupStore
|
||||
@@ -44,25 +43,25 @@ public actor PolishingService {
|
||||
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { throw PolishError.noTranscript }
|
||||
|
||||
// Mode-aware short-circuit. When the user has selected "Off", the
|
||||
// keyboard must never hit the network — we return the trimmed
|
||||
// input as-is. This is the same value the view controller would
|
||||
// produce if it skipped `polish()` entirely, but having the
|
||||
// guarantee at the service layer means future call sites (CLI,
|
||||
// tests, alternate keyboards) inherit it for free.
|
||||
if store.modeId == "off" {
|
||||
// Local engine: ASR-only — no on-device or cloud polish.
|
||||
if store.engineMode == "local" {
|
||||
return trimmed
|
||||
}
|
||||
|
||||
return try await polishRemote(trimmed)
|
||||
}
|
||||
|
||||
private func polishRemote(_ trimmed: String) async throws -> String {
|
||||
let client = injectedClient ?? store.makeClient()
|
||||
let prompt = store.systemPrompt
|
||||
let budget = effectiveTimeout(for: trimmed)
|
||||
|
||||
return try await withThrowingTaskGroup(of: String.self) { group in
|
||||
group.addTask {
|
||||
try await client.polish(trimmed, systemPrompt: prompt)
|
||||
}
|
||||
group.addTask {
|
||||
try await Task.sleep(nanoseconds: UInt64(self.timeout * 1_000_000_000))
|
||||
try await Task.sleep(nanoseconds: UInt64(budget * 1_000_000_000))
|
||||
throw PolishError.timeout
|
||||
}
|
||||
let result = try await group.next()!
|
||||
@@ -70,4 +69,10 @@ public actor PolishingService {
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Scale polish budget with transcript length (3-minute Flow utterances).
|
||||
private func effectiveTimeout(for text: String) -> TimeInterval {
|
||||
let scaled = timeout + (Double(text.count) / 200.0) * 2.0
|
||||
return min(max(scaled, timeout), 120)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
// ProgressiveDictationTranscriptAccumulator.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Merges progressive `DictationTranscriber` results into one transcript.
|
||||
// Short-form presets may emit a new time range after ~30 s; treating the
|
||||
// latest partial as the full transcript drops earlier segments.
|
||||
|
||||
import Foundation
|
||||
import CoreMedia
|
||||
|
||||
/// Combines volatile partials and finalized segments from
|
||||
/// `DictationTranscriber.results` into a single growing transcript.
|
||||
public struct ProgressiveDictationTranscriptAccumulator: Sendable {
|
||||
|
||||
private struct Segment: Sendable {
|
||||
let startSeconds: Double
|
||||
var text: String
|
||||
}
|
||||
|
||||
private var segments: [Segment] = []
|
||||
private var lastEmitted = ""
|
||||
|
||||
public init() {}
|
||||
|
||||
/// Ingest one analyzer result. Returns a non-nil full transcript when the
|
||||
/// composed text changed since the previous emission.
|
||||
public mutating func ingest(range: CMTimeRange, text: String) -> String? {
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return nil }
|
||||
|
||||
let start = range.start.seconds
|
||||
|
||||
if let idx = segments.lastIndex(where: { abs($0.startSeconds - start) < 0.001 }) {
|
||||
// Same audio window — volatile refinement of the current segment.
|
||||
segments[idx].text = trimmed
|
||||
} else if let last = segments.last,
|
||||
trimmed.hasPrefix(last.text) || last.text.hasPrefix(trimmed) {
|
||||
// Cumulative progressive update without a range change.
|
||||
let longer = trimmed.count >= last.text.count ? trimmed : last.text
|
||||
segments[segments.count - 1].text = longer
|
||||
} else {
|
||||
// New time range — append instead of replacing earlier speech.
|
||||
segments.append(Segment(startSeconds: start, text: trimmed))
|
||||
}
|
||||
|
||||
let full = composedText()
|
||||
guard full != lastEmitted else { return nil }
|
||||
lastEmitted = full
|
||||
return full
|
||||
}
|
||||
|
||||
/// Final composed transcript after the results stream finishes.
|
||||
public mutating func finalize() -> String {
|
||||
let full = composedText()
|
||||
lastEmitted = full
|
||||
return full
|
||||
}
|
||||
|
||||
private func composedText() -> String {
|
||||
segments.reduce(into: "") { partial, segment in
|
||||
partial = DictationTextComposer.compose(anchor: partial, live: segment.text)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,9 +6,12 @@
|
||||
import Foundation
|
||||
|
||||
public enum ProviderDisplayName {
|
||||
public static func name(for providerId: String) -> String {
|
||||
public static func name(
|
||||
for providerId: String,
|
||||
language: AppUILanguage? = nil
|
||||
) -> String {
|
||||
let key = "provider.\(providerId)"
|
||||
let localized = NSLocalizedString(key, comment: "")
|
||||
let localized = SharedL10n.string(key, language: language)
|
||||
if localized != key { return localized }
|
||||
return LLMProvider.provider(id: providerId).name
|
||||
}
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
// UtteranceStreamChunker.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Splits a Flow utterance PCM stream into ASR-sized chunks. When possible,
|
||||
// extends slightly past the max window to the next pause instead of cutting
|
||||
// mid-word.
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum UtteranceStreamChunker {
|
||||
|
||||
/// Yields chunks as audio arrives; the final chunk is marked `isLast`.
|
||||
public static func chunks(
|
||||
from stream: AsyncStream<AudioBufferSnapshot>,
|
||||
config: FlowUtteranceChunkConfig = .flowDefault
|
||||
) -> AsyncStream<UtteranceAudioChunk> {
|
||||
AsyncStream { continuation in
|
||||
let task = Task {
|
||||
var buffer: [Float] = []
|
||||
buffer.reserveCapacity(config.maxChunkSamples + config.pauseExtensionSamples)
|
||||
var chunkIndex = 0
|
||||
|
||||
func emit(upTo splitEnd: Int, isLast: Bool) {
|
||||
guard splitEnd > 0, splitEnd <= buffer.count else { return }
|
||||
let chunkSamples = Array(buffer[..<splitEnd])
|
||||
continuation.yield(
|
||||
UtteranceAudioChunk(index: chunkIndex, samples: chunkSamples, isLast: isLast)
|
||||
)
|
||||
chunkIndex += 1
|
||||
if splitEnd >= buffer.count {
|
||||
buffer.removeAll(keepingCapacity: true)
|
||||
} else {
|
||||
let overlapStart = max(0, splitEnd - config.overlapSamples)
|
||||
buffer = Array(buffer[overlapStart...])
|
||||
}
|
||||
}
|
||||
|
||||
for await snap in stream {
|
||||
if Task.isCancelled { break }
|
||||
guard !snap.samples.isEmpty else { continue }
|
||||
buffer.append(contentsOf: snap.samples)
|
||||
|
||||
while buffer.count >= config.maxChunkSamples {
|
||||
let split = pauseAwareSplitIndex(in: buffer, config: config)
|
||||
emit(upTo: split, isLast: false)
|
||||
}
|
||||
}
|
||||
|
||||
if !buffer.isEmpty {
|
||||
emit(upTo: buffer.count, isLast: true)
|
||||
} else if chunkIndex == 0 {
|
||||
// Empty utterance — no chunks.
|
||||
} else {
|
||||
// Stream ended exactly on boundary; mark prior path complete.
|
||||
}
|
||||
|
||||
continuation.finish()
|
||||
}
|
||||
|
||||
continuation.onTermination = { _ in
|
||||
task.cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pick a split index at or after `maxChunkSamples`, preferring a pause.
|
||||
static func pauseAwareSplitIndex(
|
||||
in buffer: [Float],
|
||||
config: FlowUtteranceChunkConfig
|
||||
) -> Int {
|
||||
let minSplit = config.maxChunkSamples
|
||||
guard buffer.count >= minSplit else { return buffer.count }
|
||||
|
||||
let searchEnd = min(buffer.count, minSplit + config.pauseExtensionSamples)
|
||||
if searchEnd <= minSplit {
|
||||
return minSplit
|
||||
}
|
||||
|
||||
let windowSize = max(config.sampleRate / 50, 160) // ~20 ms
|
||||
var bestPause: Int?
|
||||
var idx = minSplit
|
||||
while idx + windowSize <= searchEnd {
|
||||
if rms(of: buffer, start: idx, count: windowSize) < config.pauseRMSThreshold {
|
||||
bestPause = idx + windowSize
|
||||
}
|
||||
idx += windowSize / 2
|
||||
}
|
||||
|
||||
return bestPause ?? minSplit
|
||||
}
|
||||
|
||||
static func rms(of samples: [Float], start: Int, count: Int) -> Float {
|
||||
guard start >= 0, count > 0, start + count <= samples.count else { return 1 }
|
||||
var sum: Float = 0
|
||||
for i in start..<(start + count) {
|
||||
let v = samples[i]
|
||||
sum += v * v
|
||||
}
|
||||
return sqrtf(sum / Float(count))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// UtteranceTranscriptStitcher.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Orders pipelined chunk transcripts and merges overlap at boundaries.
|
||||
|
||||
import Foundation
|
||||
|
||||
public struct UtteranceTranscriptStitcher: Sendable {
|
||||
private var segments: [(index: Int, text: String)] = []
|
||||
|
||||
public init() {}
|
||||
|
||||
public mutating func append(index: Int, text: String) {
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return }
|
||||
if let existing = segments.firstIndex(where: { $0.index == index }) {
|
||||
segments[existing].text = trimmed
|
||||
} else {
|
||||
segments.append((index, trimmed))
|
||||
segments.sort { $0.index < $1.index }
|
||||
}
|
||||
}
|
||||
|
||||
public func composed() -> String {
|
||||
guard let first = segments.first else { return "" }
|
||||
var result = first.text
|
||||
for segment in segments.dropFirst() {
|
||||
result = Self.mergeWithOverlap(previous: result, next: segment.text)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/// Merge `next` onto `previous`, dropping duplicated suffix/prefix overlap.
|
||||
public static func mergeWithOverlap(previous: String, next: String) -> String {
|
||||
let trimmedNext = next.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmedNext.isEmpty else { return previous }
|
||||
guard !previous.isEmpty else { return trimmedNext }
|
||||
|
||||
// Character-granular probe — works for CJK without word boundaries.
|
||||
let prevChars = Array(previous)
|
||||
let nextChars = Array(trimmedNext)
|
||||
let maxProbe = min(64, prevChars.count, nextChars.count)
|
||||
if maxProbe > 0 {
|
||||
for length in stride(from: maxProbe, through: 1, by: -1) {
|
||||
let suffix = prevChars.suffix(length)
|
||||
let prefix = nextChars.prefix(length)
|
||||
if suffix.elementsEqual(prefix) {
|
||||
return previous + String(nextChars.dropFirst(length))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Punctuation-insensitive CJK overlap (e.g. "很好," + "很好继续").
|
||||
let normalizedPrev = normalizeForOverlap(previous)
|
||||
let normalizedNext = normalizeForOverlap(trimmedNext)
|
||||
let nPrev = Array(normalizedPrev)
|
||||
let nNext = Array(normalizedNext)
|
||||
let normProbe = min(64, nPrev.count, nNext.count)
|
||||
if normProbe > 0 {
|
||||
for length in stride(from: normProbe, through: 2, by: -1) {
|
||||
if nPrev.suffix(length).elementsEqual(nNext.prefix(length)) {
|
||||
// Map normalized overlap length back to raw `next` drop count.
|
||||
let drop = overlapDropCount(in: trimmedNext, normalizedPrefixLength: length)
|
||||
return previous + String(trimmedNext.dropFirst(drop))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// English / spaced languages.
|
||||
let maxWordProbe = min(6, previous.split(separator: " ").count, trimmedNext.split(separator: " ").count)
|
||||
if maxWordProbe > 0 {
|
||||
let prevWords = previous.split(separator: " ", omittingEmptySubsequences: true)
|
||||
let nextWords = trimmedNext.split(separator: " ", omittingEmptySubsequences: true)
|
||||
for wordCount in stride(from: maxWordProbe, through: 1, by: -1) {
|
||||
if prevWords.suffix(wordCount).elementsEqual(nextWords.prefix(wordCount)) {
|
||||
let mergedPrefix = nextWords.dropFirst(wordCount).joined(separator: " ")
|
||||
if mergedPrefix.isEmpty { return previous }
|
||||
if previous.last == " " || previous.last == "\n" {
|
||||
return previous + mergedPrefix
|
||||
}
|
||||
return previous + " " + mergedPrefix
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return DictationTextComposer.compose(anchor: previous, live: trimmedNext)
|
||||
}
|
||||
|
||||
private static func normalizeForOverlap(_ text: String) -> String {
|
||||
text.unicodeScalars.filter {
|
||||
!CharacterSet.whitespacesAndNewlines.contains($0)
|
||||
&& !CharacterSet.punctuationCharacters.contains($0)
|
||||
}.map { Character($0) }.reduce(into: "") { $0.append($1) }
|
||||
}
|
||||
|
||||
/// How many raw characters to drop from `next` given a normalized-prefix overlap length.
|
||||
private static func overlapDropCount(in next: String, normalizedPrefixLength: Int) -> Int {
|
||||
var normalizedCount = 0
|
||||
var rawIndex = next.startIndex
|
||||
while rawIndex < next.endIndex, normalizedCount < normalizedPrefixLength {
|
||||
let scalar = next[rawIndex]
|
||||
if !scalar.isWhitespace, !scalar.isPunctuation {
|
||||
normalizedCount += 1
|
||||
}
|
||||
rawIndex = next.index(after: rawIndex)
|
||||
}
|
||||
return next.distance(from: next.startIndex, to: rawIndex)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/* Engine status labels */
|
||||
"engine.summary.local" = "On-device · %@";
|
||||
"engine.summary.cloud" = "Active: %@";
|
||||
"engine.summary.cloudWithModel" = "Active: %1$@ · %2$@";
|
||||
"engine.asr.appleSpeech" = "Apple SpeechAnalyzer";
|
||||
"model.qwen3asr.name" = "Qwen3-ASR 0.6B (CoreML)";
|
||||
|
||||
/* LLM providers */
|
||||
"provider.openai" = "OpenAI";
|
||||
"provider.deepseek" = "DeepSeek";
|
||||
"provider.qwen" = "Qwen (DashScope)";
|
||||
"provider.zhipu" = "Zhipu GLM";
|
||||
"provider.moonshot" = "Moonshot";
|
||||
"provider.custom" = "Custom";
|
||||
|
||||
/* LLM errors */
|
||||
"error.llm.invalidURL" = "Invalid API URL. Check Base URL in Settings.";
|
||||
"error.llm.noAPIKey" = "API Key is missing.";
|
||||
"error.llm.http" = "API returned HTTP %lld. Try again later or contact the provider.";
|
||||
"error.llm.decoding" = "Failed to parse the API response.";
|
||||
"error.llm.transport" = "Network error. Check your connection and try again.";
|
||||
"error.llm.rateLimited" = "Too many API requests. Please wait and try again.";
|
||||
"error.llm.cancelled" = "Request cancelled.";
|
||||
|
||||
/* ASR errors */
|
||||
"error.asr.localeUnsupported" = "Speech language assets are unavailable. Try again later or switch the recognition language.";
|
||||
"error.asr.assetsNotReady" = "Speech language assets are not ready. Try again later.";
|
||||
"error.asr.formatUnsupported" = "This device does not support the required audio format.";
|
||||
"error.asr.noSpeech" = "No speech detected. Please try again.";
|
||||
"error.asr.chunkFailed" = "Segment %lld failed: %@";
|
||||
@@ -0,0 +1,30 @@
|
||||
/* Engine status labels */
|
||||
"engine.summary.local" = "本地 · %@";
|
||||
"engine.summary.cloud" = "当前:%@";
|
||||
"engine.summary.cloudWithModel" = "当前:%1$@ · %2$@";
|
||||
"engine.asr.appleSpeech" = "Apple 语音识别";
|
||||
"model.qwen3asr.name" = "Qwen3-ASR 0.6B (CoreML)";
|
||||
|
||||
/* LLM providers */
|
||||
"provider.openai" = "OpenAI";
|
||||
"provider.deepseek" = "DeepSeek";
|
||||
"provider.qwen" = "通义千问";
|
||||
"provider.zhipu" = "智谱 GLM";
|
||||
"provider.moonshot" = "月之暗面";
|
||||
"provider.custom" = "自定义";
|
||||
|
||||
/* LLM errors */
|
||||
"error.llm.invalidURL" = "API 地址无效。请在设置中检查 Base URL。";
|
||||
"error.llm.noAPIKey" = "未填写 API Key。";
|
||||
"error.llm.http" = "API 返回 HTTP %lld。请稍后重试或联系服务方。";
|
||||
"error.llm.decoding" = "解析 API 响应失败。";
|
||||
"error.llm.transport" = "网络错误,请检查连接后重试。";
|
||||
"error.llm.rateLimited" = "API 调用过于频繁,请稍候再试。";
|
||||
"error.llm.cancelled" = "请求已取消。";
|
||||
|
||||
/* ASR errors */
|
||||
"error.asr.localeUnsupported" = "当前系统未分配可用语音语言模型,请稍后重试或切换语言。";
|
||||
"error.asr.assetsNotReady" = "语音语言资源未就绪,请稍后重试。";
|
||||
"error.asr.formatUnsupported" = "当前设备不支持该语音输入格式。";
|
||||
"error.asr.noSpeech" = "未识别到语音内容,请重试。";
|
||||
"error.asr.chunkFailed" = "第 %lld 段识别失败:%@";
|
||||
Reference in New Issue
Block a user