feat: migrate on-device Qwen3 ASR to CoreML for background Flow dictation

Replace MLX GPU inference with CoreML bundles so transcription continues while the host app is backgrounded. Adds model download and warm-up, vendored Qwen3Speech, and updates onboarding, settings, and copy for the ~1.6 GB CoreML package (iOS 18+).
2026-06-23 00:46:58 +08:00
parent 5e5122f172
commit df1c5ff32c
160 changed files with 22080 additions and 492 deletions
@@ -0,0 +1,87 @@
+// AppUILanguage.swift
+// OSGKeyboard · Shared
+//
+// In-app UI language override (main app + keyboard extension strings).
+// Distinct from `localeId`, which controls speech recognition language.
+
+import Foundation
+
+public enum AppUILanguage: String, CaseIterable, Identifiable, Sendable, Codable {
+    case auto
+    case english = "en"
+    case chinese = "zh-Hans"
+
+    public var id: String { rawValue }
+
+    public var labelKey: String {
+        switch self {
+        case .auto:    return "settings.appLanguage.auto"
+        case .english: return "settings.appLanguage.english"
+        case .chinese: return "settings.appLanguage.chinese"
+        }
+    }
+
+    /// Locale for SwiftUI `.environment(\.locale, …)` in the host app.
+    public var swiftUILocale: Locale {
+        switch self {
+        case .auto:
+            return Locale.autoupdatingCurrent
+        case .english:
+            return Locale(identifier: "en")
+        case .chinese:
+            return Locale(identifier: "zh-Hans")
+        }
+    }
+
+    /// `.lproj` folder name used for manual bundle lookups (extension).
+    public func resolvedLanguageCode(
+        preferredLanguages: [String] = Locale.preferredLanguages
+    ) -> String {
+        switch self {
+        case .english:
+            return "en"
+        case .chinese:
+            return "zh-Hans"
+        case .auto:
+            if preferredLanguages.contains(where: { $0.hasPrefix("zh") }) {
+                return "zh-Hans"
+            }
+            return "en"
+        }
+    }
+
+    public static func fromStored(_ raw: String?) -> AppUILanguage {
+        guard let raw, let value = AppUILanguage(rawValue: raw) else { return .auto }
+        return value
+    }
+
+    /// Picks the best-matching `.lproj` inside `container` for this preference.
+    public static func localizedBundle(
+        in container: Bundle,
+        language: AppUILanguage,
+        preferredLanguages: [String] = Locale.preferredLanguages
+    ) -> Bundle {
+        let code = language.resolvedLanguageCode(preferredLanguages: preferredLanguages)
+        guard let path = container.path(forResource: code, ofType: "lproj"),
+              let bundle = Bundle(path: path) else {
+            return container
+        }
+        return bundle
+    }
+
+    public static func localizedString(
+        _ key: String,
+        tableName: String?,
+        bundle container: Bundle,
+        language: AppUILanguage = AppGroupStore().uiLanguage
+    ) -> String {
+        let bundle = localizedBundle(in: container, language: language)
+        return NSLocalizedString(
+            key,
+            tableName: tableName,
+            bundle: bundle,
+            value: key,
+            comment: ""
+        )
+    }
+}
@@ -9,18 +9,37 @@ public enum EngineServiceLabel {
    public static func summary(
        engineMode: String,
        providerId: String,
-        model: String
+        model: String,
+        localASRBackend: LocalASRBackend = .speechAnalyzer,
+        language: AppUILanguage? = nil
    ) -> String {
-        let isChinese = Locale.preferredLanguages.first?.hasPrefix("zh") == true
-        let prefix = isChinese ? "当前：" : "Active: "
+        let lang = language ?? AppGroupStore().uiLanguage
        if engineMode == "local" {
-            return isChinese
-                ? "\(prefix)本地引擎 · Apple SpeechAnalyzer"
-                : "\(prefix)On-device · Apple SpeechAnalyzer"
+            let asrName = asrDisplayName(for: localASRBackend, language: lang)
+            return SharedL10n.format("engine.summary.local", language: lang, asrName)
        }
-        let providerName = ProviderDisplayName.name(for: providerId)
+        let providerName = ProviderDisplayName.name(for: providerId, language: lang)
        let trimmedModel = model.trimmingCharacters(in: .whitespacesAndNewlines)
-        if trimmedModel.isEmpty { return "\(prefix)\(providerName)" }
-        return "\(prefix)\(providerName) · \(trimmedModel)"
+        if trimmedModel.isEmpty {
+            return SharedL10n.format("engine.summary.cloud", language: lang, providerName)
+        }
+        return SharedL10n.format(
+            "engine.summary.cloudWithModel",
+            language: lang,
+            providerName,
+            trimmedModel
+        )
+    }
+
+    private static func asrDisplayName(
+        for backend: LocalASRBackend,
+        language: AppUILanguage
+    ) -> String {
+        switch backend {
+        case .speechAnalyzer:
+            return SharedL10n.string("engine.asr.appleSpeech", language: language)
+        case .qwen3ASR:
+            return SharedL10n.string("model.qwen3asr.name", language: language)
+        }
    }
 }
@@ -0,0 +1,69 @@
+// FlowUtteranceChunkConfig.swift
+// OSGKeyboard · Shared
+//
+// Chunking policy for pipelined Flow utterance ASR (up to 3 minutes).
+
+import Foundation
+
+public struct FlowUtteranceChunkConfig: Sendable, Equatable {
+    /// Target maximum duration per ASR chunk.
+    public let maxChunkDurationSeconds: TimeInterval
+    /// Tail overlap fed into the next chunk for boundary dedup when stitching.
+    public let overlapDurationSeconds: TimeInterval
+    /// After hitting the max window, wait up to this long for a pause before hard-splitting.
+    public let pauseExtensionMaxSeconds: TimeInterval
+    /// RMS below this is treated as a pause candidate (Float32 mono @ 16 kHz).
+    public let pauseRMSThreshold: Float
+    public let sampleRate: Int
+
+    public init(
+        maxChunkDurationSeconds: TimeInterval,
+        overlapDurationSeconds: TimeInterval,
+        pauseExtensionMaxSeconds: TimeInterval,
+        pauseRMSThreshold: Float,
+        sampleRate: Int
+    ) {
+        self.maxChunkDurationSeconds = maxChunkDurationSeconds
+        self.overlapDurationSeconds = overlapDurationSeconds
+        self.pauseExtensionMaxSeconds = pauseExtensionMaxSeconds
+        self.pauseRMSThreshold = pauseRMSThreshold
+        self.sampleRate = sampleRate
+    }
+
+    public var maxChunkSamples: Int {
+        Int(maxChunkDurationSeconds * Double(sampleRate))
+    }
+
+    public var overlapSamples: Int {
+        Int(overlapDurationSeconds * Double(sampleRate))
+    }
+
+    public var pauseExtensionSamples: Int {
+        Int(pauseExtensionMaxSeconds * Double(sampleRate))
+    }
+
+    /// Default for keyboard Flow utterances (≤ 3 min, pipelined ASR).
+    public static let flowDefault = FlowUtteranceChunkConfig(
+        maxChunkDurationSeconds: 30,
+        overlapDurationSeconds: 0.5,
+        pauseExtensionMaxSeconds: 2,
+        pauseRMSThreshold: 0.015,
+        sampleRate: 16_000
+    )
+}
+
+public struct UtteranceAudioChunk: Sendable, Equatable {
+    public let index: Int
+    public let samples: [Float]
+    public let isLast: Bool
+
+    public init(index: Int, samples: [Float], isLast: Bool) {
+        self.index = index
+        self.samples = samples
+        self.isLast = isLast
+    }
+
+    public var durationSeconds: Double {
+        Double(samples.count) / 16_000.0
+    }
+}
@@ -0,0 +1,56 @@
+// LocalASRBackend.swift
+// OSGKeyboard · Shared
+//
+// Identifies which on-device speech recognition engine to use when the
+// user picks the "local" engine (no cloud LLM polish). The shared
+// factory `ASRServiceFactory` dispatches on this enum; the settings UI
+// renders it as a picker.
+//
+// Why an enum in `Shared` rather than living next to the concrete
+// `ASRService` implementations: the value must be serialisable into
+// the App Group store (so the keyboard extension can observe the
+// selection), exposed via `ProviderConfig` (UI binding) and consumed
+// by every layer that asks for an ASR backend.
+
+import Foundation
+
+public enum LocalASRBackend: String, CaseIterable, Identifiable, Sendable, Codable {
+    /// iOS 26 `SpeechAnalyzer` + `DictationTranscriber`. Always
+    /// on-device, no asset download, ships with iOS. Default for every
+    /// fresh install — anything else is opt-in.
+    case speechAnalyzer
+
+    /// Qwen3-ASR-0.6B via CoreML (Neural Engine + CPU). Stronger on Chinese
+    /// dialects and noisy audio than `SpeechAnalyzer`, works in Flow while
+    /// the host app is backgrounded, but requires a ~1.6 GB download on first
+    /// use and iOS 18+.
+    case qwen3ASR
+
+    public var id: String { rawValue }
+
+    /// Localisation key for the human label in the settings picker.
+    public var labelKey: String {
+        switch self {
+        case .speechAnalyzer: return "asr.backend.speechAnalyzer.label"
+        case .qwen3ASR:        return "asr.backend.qwen3.label"
+        }
+    }
+
+    /// Localisation key for the one-line subtitle shown under the label.
+    public var blurbKey: String {
+        switch self {
+        case .speechAnalyzer: return "asr.backend.speechAnalyzer.blurb"
+        case .qwen3ASR:        return "asr.backend.qwen3.blurb"
+        }
+    }
+
+    /// Whether this backend needs the user to download a model file
+    /// before it can run. Used to gate the "Downloading Qwen3-ASR" UI
+    /// in a follow-up; for now we just expose the flag.
+    public var requiresModelDownload: Bool {
+        switch self {
+        case .speechAnalyzer: return false
+        case .qwen3ASR:        return true
+        }
+    }
+}
@@ -0,0 +1,54 @@
+// OnDeviceModel.swift
+// OSGKeyboard · Shared
+//
+// Identity of an on-device model the host app downloads and the
+// keyboard extension observes via App Group flags (the extension
+// cannot read the main app's Caches directory).
+
+import Foundation
+
+public enum OnDeviceModel: String, CaseIterable, Identifiable, Sendable {
+    case qwen3ASR
+
+    public var id: String { rawValue }
+
+    /// CoreML inference bundle (`aufklarer/Qwen3-ASR-CoreML`), derived from
+    /// official `Qwen/Qwen3-ASR-0.6B`.
+    public var repoId: String {
+        switch self {
+        case .qwen3ASR: return "aufklarer/Qwen3-ASR-CoreML"
+        }
+    }
+
+    /// Tokenizer files (vocab / merges) pulled from the upstream Qwen repo.
+    public var tokenizerRepoId: String {
+        switch self {
+        case .qwen3ASR: return "Qwen/Qwen3-ASR-0.6B"
+        }
+    }
+
+    public var displayName: String {
+        switch self {
+        case .qwen3ASR: return "Qwen3-ASR 0.6B (CoreML)"
+        }
+    }
+
+    public var approximateSizeMB: Int {
+        switch self {
+        case .qwen3ASR: return 1_600
+        }
+    }
+
+    public var compactSizeLabel: String {
+        "\(approximateSizeMB)M"
+    }
+
+    /// Settings list title: model name plus compact size.
+    public var listTitle: String {
+        "\(displayName) · \(compactSizeLabel)"
+    }
+
+    public var repoAndSizeLabel: String {
+        "\(repoId) · \(approximateSizeMB) MB"
+    }
+}
@@ -30,6 +30,11 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
        static let hasCompletedOnboarding = "config.hasCompletedOnboarding"
        static let onboardingPage         = "config.onboardingPage"
        static let hasAcknowledgedCloudSharing = "config.hasAcknowledgedCloudSharing"
+        // Which on-device ASR engine to use when `engineMode == "local"`.
+        // Persisted in the App Group so the keyboard can read the
+        // selection even though it never instantiates the backend itself.
+        static let localASRBackend = "config.localASRBackend"
+        static let uiLanguage = "config.uiLanguage"
    }

    @Published public var providerId: String {
@@ -64,8 +69,8 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
    @Published public var localeId: String {
        didSet { defaults.set(localeId, forKey: Key.localeId) }
    }
-    /// "local" → on-device ASR only, no LLM polishing.
-    /// "cloud" → ASR + LLM polish (default).
+    /// "local" → on-device ASR only (raw transcript delivery).
+    /// "cloud" → ASR + LLM polish (always on; modeId kept for compatibility).
    @Published public var engineMode: String {
        didSet { defaults.set(engineMode, forKey: Key.engineMode) }
    }
@@ -85,6 +90,16 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
    @Published public var hasAcknowledgedCloudSharing: Bool {
        didSet { defaults.set(hasAcknowledgedCloudSharing, forKey: Key.hasAcknowledgedCloudSharing) }
    }
+    /// Which on-device ASR engine backs the "local" engine mode. Only
+    /// consulted when `isLocalEngine == true`; the cloud engine always
+    /// uses `SpeechAnalyzer`.
+    @Published public var localASRBackend: LocalASRBackend {
+        didSet { defaults.set(localASRBackend.rawValue, forKey: Key.localASRBackend) }
+    }
+    /// Host-app UI language. Also mirrored to the App Group for the keyboard extension.
+    @Published public var uiLanguage: AppUILanguage {
+        didSet { defaults.set(uiLanguage.rawValue, forKey: Key.uiLanguage) }
+    }

    public var isConfigured: Bool {
        // Local engine (on-device ASR only) doesn't need an API key,
@@ -96,7 +111,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
        return !baseURL.isEmpty && !apiKey.isEmpty && !model.isEmpty
    }

-    /// On-device ASR only — no cloud LLM polish.
+    /// On-device ASR only; no cloud API required.
    public var isLocalEngine: Bool { engineMode == "local" }

    /// The system prompt the user *sees* in the editor — fall back to the
@@ -131,6 +146,19 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
        let savedPage = resolvedDefaults.integer(forKey: Key.onboardingPage)
        self.onboardingPage = savedPage > 0 ? savedPage : 0
        self.hasAcknowledgedCloudSharing = resolvedDefaults.bool(forKey: Key.hasAcknowledgedCloudSharing)
+        // Tolerate missing / unknown raw values (e.g. an enum case that
+        // was renamed in a later build) by falling back to the default
+        // rather than crashing inside `RawRepresentable.init`.
+        let rawBackend = resolvedDefaults.string(forKey: Key.localASRBackend) ?? LocalASRBackend.speechAnalyzer.rawValue
+        self.localASRBackend = LocalASRBackend(rawValue: rawBackend) ?? .speechAnalyzer
+        self.uiLanguage = AppUILanguage.fromStored(
+            resolvedDefaults.string(forKey: Key.uiLanguage)
+        )
+
+        // Cloud no longer exposes off/transcribe; migrate legacy values.
+        if self.engineMode == "cloud", self.modeId != "polish" {
+            self.modeId = "polish"
+        }
    }

    /// Read the API key from the Keychain, falling back to a one-time
@@ -0,0 +1,17 @@
+// TranscriptionDelivery.swift
+// OSGKeyboard · Shared
+//
+// Host-app → keyboard handoff payload: final text plus an optional soft
+// warning when cloud polish failed but the raw transcript is still delivered.
+
+import Foundation
+
+public struct TranscriptionDelivery: Sendable, Equatable {
+    public let text: String
+    public let polishWarning: String?
+
+    public init(text: String, polishWarning: String? = nil) {
+        self.text = text
+        self.polishWarning = polishWarning
+    }
+}