feat: migrate on-device Qwen3 ASR to CoreML for background Flow dictation

Replace MLX GPU inference with CoreML bundles so transcription continues while the host app is backgrounded. Adds model download and warm-up, vendored Qwen3Speech, and updates onboarding, settings, and copy for the ~1.6 GB CoreML package (iOS 18+).
2026-06-23 00:46:58 +08:00
parent 5e5122f172
commit df1c5ff32c
160 changed files with 22080 additions and 492 deletions
@@ -28,7 +28,16 @@ public final class KeyboardViewController: UIInputViewController {
        static let pollIntervalNs: UInt64 = 200_000_000
        /// Give the user time to manually open the host app when auto-jump fails.
        static let startTimeout: TimeInterval = 30
-        static let resultTimeout: TimeInterval = 45
+
+        static func resultTimeout(
+            engineMode: String,
+            localASRBackend: LocalASRBackend
+        ) -> TimeInterval {
+            FlowSessionKeys.keyboardResultTimeout(
+                engineMode: engineMode,
+                localASRBackend: localASRBackend
+            )
+        }
    }

    private enum DictationWatchdog {
@@ -128,6 +137,7 @@ public final class KeyboardViewController: UIInputViewController {
        state.setMode             = { [weak self] m in self?.persistMode(m) }
        state.setLocale           = { [weak self] l in self?.persistLocale(l) }
        state.setEngineMode       = { [weak self] m in self?.persistEngineMode(m) }
+        state.setLocalASRBackend  = { [weak self] b in self?.persistLocalASRBackend(b) }
        state.insertNewline       = { [weak self] in self?.textDocumentProxy.insertText("\n") }
        state.insertSpace         = { [weak self] in self?.textDocumentProxy.insertText(" ") }
        state.deleteBackward      = { [weak self] in self?.textDocumentProxy.deleteBackward() }
@@ -191,6 +201,9 @@ public final class KeyboardViewController: UIInputViewController {
    }

    private func refreshFlowSessionState() {
+        persistor.refreshRuntimeFlags(into: state)
+        consumePendingFlowDeliveryIfNeeded()
+
        let active = FlowSessionBridge.isSessionActive()
        state.flowSessionActive = active

@@ -209,11 +222,33 @@ public final class KeyboardViewController: UIInputViewController {
        }
    }

+    /// Pick up transcripts/errors the host wrote while the extension was paused.
+    private func consumePendingFlowDeliveryIfNeeded() {
+        if isAwaitingFlowResult {
+            if let delivery = FlowSessionBridge.consumeTranscriptionDelivery() {
+                isAwaitingFlowResult = false
+                stopFlowWatchdog()
+                handleFlowTranscript(delivery)
+                return
+            }
+            if let error = FlowSessionBridge.consumeTranscriptionError() {
+                isAwaitingFlowResult = false
+                stopFlowWatchdog()
+                state.phase = .error(.unknown(error), message: error)
+                scheduleAutoClearError()
+                return
+            }
+        }
+
+        if isPendingFlowStart, FlowSessionBridge.isSessionActive() {
+            completeFlowStartHandoff()
+        }
+    }
+
    /// When the host session is down, proactively jump to the app to start it.
    private func maybeAutoStartFlowSession() {
        guard !FlowSessionBridge.isSessionActive() else { return }
        guard !isPendingFlowStart, !isFlowRecording, !isAwaitingFlowResult else { return }
-        guard state.mode != .off else { return }
        guard hasFullAccess, AppGroup.isAvailable else { return }
        guard case .idle = state.phase else { return }

@@ -249,7 +284,6 @@ public final class KeyboardViewController: UIInputViewController {
        default:
            return
        }
-        guard state.mode != .off else { return }
        guard hasFullAccess else {
            let msg = ExtL10n.string("keyboard.error.fullAccessRequired")
            state.phase = .error(.unknown(msg), message: msg)
@@ -392,12 +426,16 @@ public final class KeyboardViewController: UIInputViewController {
        stopFlowWatchdog()
        isAwaitingFlowResult = true
        let startedAt = Date().timeIntervalSince1970
+        let resultTimeout = FlowWatchdog.resultTimeout(
+            engineMode: state.engineMode,
+            localASRBackend: state.localASRBackend
+        )
        flowWatchdogTask = Task { @MainActor [weak self] in
            while let self, !Task.isCancelled {
-                if let result = FlowSessionBridge.consumeTranscriptionResult() {
+                if let delivery = FlowSessionBridge.consumeTranscriptionDelivery() {
                    self.isAwaitingFlowResult = false
                    self.stopFlowWatchdog()
-                    self.handleFlowTranscript(result)
+                    self.handleFlowTranscript(delivery)
                    return
                }
                if let error = FlowSessionBridge.consumeTranscriptionError() {
@@ -408,7 +446,7 @@ public final class KeyboardViewController: UIInputViewController {
                    return
                }
                let now = Date().timeIntervalSince1970
-                if now - startedAt > FlowWatchdog.resultTimeout {
+                if now - startedAt > resultTimeout {
                    self.isAwaitingFlowResult = false
                    self.stopFlowWatchdog()
                    let msg = ExtL10n.string("keyboard.flow.resultTimeout")
@@ -421,8 +459,8 @@ public final class KeyboardViewController: UIInputViewController {
        }
    }

-    private func handleFlowTranscript(_ transcript: String) {
-        let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
+    private func handleFlowTranscript(_ delivery: TranscriptionDelivery) {
+        let trimmed = delivery.text.trimmingCharacters(in: .whitespacesAndNewlines)
        guard !trimmed.isEmpty else {
            state.phase = .idle
            state.level = 0
@@ -432,7 +470,12 @@ public final class KeyboardViewController: UIInputViewController {
        textDocumentProxy.insertText(trimmed)
        state.lastTranscript = ""
        state.level = 0
-        state.phase = .idle
+        if let warning = delivery.polishWarning {
+            state.phase = .error(.unknown(warning), message: warning)
+            scheduleAutoClearError()
+        } else {
+            state.phase = .idle
+        }
        debug("flow insert length=\(trimmed.count)")
    }

@@ -457,8 +500,8 @@ public final class KeyboardViewController: UIInputViewController {
        }
    }

-    private func handleFinalTranscript(_ transcript: String) {
-        let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
+    private func handleFinalTranscript(_ delivery: TranscriptionDelivery) {
+        let trimmed = delivery.text.trimmingCharacters(in: .whitespacesAndNewlines)
        guard !trimmed.isEmpty else {
            debug("received empty transcript")
            awaitingDictationResult = false
@@ -469,14 +512,19 @@ public final class KeyboardViewController: UIInputViewController {
        debug("received transcript length=\(trimmed.count)")
        awaitingDictationResult = false
        stopDictationWatchdog()
-        // Local engine or transcribe mode: insert directly, no LLM call.
-        if state.isLocalEngine || state.mode == .transcribe {
+        // Local engine: host app delivers raw ASR transcript; insert as-is.
+        if state.isLocalEngine {
            textDocumentProxy.insertText(trimmed)
            state.lastTranscript = ""
-            state.phase = .idle
+            if let warning = delivery.polishWarning {
+                state.phase = .error(.unknown(warning), message: warning)
+                scheduleAutoClearError()
+            } else {
+                state.phase = .idle
+            }
            return
        }
-        // `.polish` (default): call the LLM.
+        // Cloud engine: always polish via the configured LLM.
        state.phase = .processing
        Task { @MainActor [weak self] in
            guard let self else { return }
@@ -555,6 +603,11 @@ public final class KeyboardViewController: UIInputViewController {
        persistor.persist(engineMode: mode)
    }

+    private func persistLocalASRBackend(_ backend: LocalASRBackend) {
+        state.localASRBackend = backend
+        persistor.persist(localASRBackend: backend)
+    }
+
    // MARK: - Open host app

    private func openHostApp(path: String = "settings") {
@@ -592,9 +645,9 @@ public final class KeyboardViewController: UIInputViewController {
    }

    private func consumePendingDictationResultIfNeeded() {
-        guard let transcript = DictationBridge.consumePendingTranscript() else { return }
+        guard let delivery = DictationBridge.consumePendingDelivery() else { return }
        debug("consumePendingDictationResultIfNeeded success")
-        handleFinalTranscript(transcript)
+        handleFinalTranscript(delivery)
    }

    private func refreshDictationProgressStateIfNeeded() {
@@ -31,8 +31,14 @@ public struct AppGroupPersistor {
        }
        let store = AppGroupStore()
        state.localeId         = store.localeId
-        state.mode             = KeyboardViewController.State.InputMode(rawValue: store.modeId) ?? .polish
+        // Both engines always polish; ignore legacy off/transcribe modeId.
+        state.mode             = .polish
        state.engineMode       = store.engineMode
+        state.localASRBackend  = store.localASRBackend
+        state.localModelsReady = OnDeviceModelStatus.isLocalStackReady(
+            asrBackend: store.localASRBackend
+        )
+        state.localModelsLoaded = OnDeviceModelStatus.modelsLoadedInMemory()

        #if DEBUG
        // Print a masked view of the live App Group config so we can see
@@ -49,17 +55,31 @@ public struct AppGroupPersistor {
        }
        print("""
        🔍 [AppGroupPersistor.load]
-           providerId = \(store.providerId)
-           baseURL    = \(store.baseURL)
-           apiKey     = \(masked)
-           model      = \(store.model)
-           modeId     = \(store.modeId)
-           localeId   = \(store.localeId)
+           providerId      = \(store.providerId)
+           baseURL         = \(store.baseURL)
+           apiKey          = \(masked)
+           model           = \(store.model)
+           modeId          = \(store.modeId)
+           localeId        = \(store.localeId)
+           localASRBackend = \(store.localASRBackend.rawValue)
        """)
        #endif
        return .loaded
    }

+    /// Lightweight refresh for flags the host app may update while the
+    /// keyboard stays open (model downloads, engine switches).
+    public func refreshRuntimeFlags(into state: KeyboardViewController.State) {
+        guard AppGroup.isAvailable else { return }
+        let store = AppGroupStore()
+        state.engineMode = store.engineMode
+        state.localASRBackend = store.localASRBackend
+        state.localModelsReady = OnDeviceModelStatus.isLocalStackReady(
+            asrBackend: store.localASRBackend
+        )
+        state.localModelsLoaded = OnDeviceModelStatus.modelsLoadedInMemory()
+    }
+
    /// Persist `mode` to the App Group store.
    public func persist(mode: KeyboardViewController.State.InputMode) {
        guard AppGroup.isAvailable else { return }
@@ -77,4 +97,10 @@ public struct AppGroupPersistor {
        guard AppGroup.isAvailable else { return }
        AppGroupStore().setEngineMode(engineMode)
    }
+
+    /// Persist `localASRBackend` to the App Group store.
+    public func persist(localASRBackend: LocalASRBackend) {
+        guard AppGroup.isAvailable else { return }
+        AppGroupStore().setLocalASRBackend(localASRBackend)
+    }
 }
@@ -7,20 +7,27 @@

 import Foundation
 import SwiftUI
+import OSGKeyboardShared

 enum ExtL10n {
    private static let table = "Keyboard"
-    private static let bundle = Bundle(for: KeyboardViewController.self)
+    private static let container = Bundle(for: KeyboardViewController.self)
+
+    private static var bundle: Bundle {
+        AppUILanguage.localizedBundle(
+            in: container,
+            language: AppGroupStore().uiLanguage
+        )
+    }

    static func string(_ key: String) -> String {
-        let value = NSLocalizedString(
+        NSLocalizedString(
            key,
            tableName: table,
            bundle: bundle,
            value: key,
            comment: ""
        )
-        return value
    }

    static func text(_ key: String) -> Text {
@@ -69,12 +69,9 @@ public struct KeyboardRootView: View {
    private var topBar: some View {
        HStack(spacing: Spacing.xs) {
            if state.isLocalEngine {
-                // Local engine: always transcribe, no mode menu needed.
                LocalEngineChip()
            } else {
-                ModeChip(mode: state.mode) { newMode in
-                    state.setMode(newMode)
-                }
+                CloudEngineChip()
            }
            LocaleChip(localeId: state.localeId) { newId in
                state.setLocale(newId)
@@ -103,6 +100,9 @@ public struct KeyboardRootView: View {
                phase: state.phase,
                transcript: state.lastTranscript,
                flowSessionActive: state.flowSessionActive,
+                isLocalEngine: state.isLocalEngine,
+                localModelsReady: state.localModelsReady,
+                localModelsLoaded: state.localModelsLoaded,
                openSettings: state.openSettings,
                startFlowSession: state.startFlowSession
            )
@@ -200,6 +200,9 @@ private struct TranscriptLine: View {
    let phase: KeyboardViewController.State.Phase
    let transcript: String
    let flowSessionActive: Bool
+    let isLocalEngine: Bool
+    let localModelsReady: Bool
+    let localModelsLoaded: Bool
    let openSettings: () -> Void
    let startFlowSession: () -> Void

@@ -207,7 +210,30 @@ private struct TranscriptLine: View {
        ZStack {
            switch phase {
            case .idle:
-                if flowSessionActive {
+                if isLocalEngine, !localModelsReady {
+                    Button(action: openSettings) {
+                        HStack(spacing: 4) {
+                            Text(ExtL10n.string("keyboard.models.notDownloaded"))
+                            Image(systemName: "chevron.right")
+                                .font(.system(size: 10, weight: .semibold))
+                        }
+                        .font(TypeStyle.caption)
+                        .foregroundStyle(palette.warning)
+                        .lineLimit(1)
+                        .truncationMode(.tail)
+                        .frame(maxWidth: .infinity)
+                        .contentShape(Rectangle())
+                    }
+                    .buttonStyle(.plain)
+                    .accessibilityHint(ExtL10n.text("keyboard.models.downloadHint"))
+                } else if isLocalEngine, localModelsReady, !localModelsLoaded {
+                    HStack(spacing: 6) {
+                        ProgressView().controlSize(.mini).tint(palette.textSecondary)
+                        ExtL10n.text("keyboard.models.warming")
+                            .font(TypeStyle.caption)
+                            .foregroundStyle(palette.textSecondary)
+                    }
+                } else if flowSessionActive {
                    ExtL10n.text("keyboard.placeholder.idle")
                        .font(TypeStyle.caption)
                        .foregroundStyle(palette.textTertiary)
@@ -393,6 +419,26 @@ private struct StatusBadge: View {
    }
 }

+// MARK: - Cloud engine chip (cloud always ASR + LLM polish)
+
+private struct CloudEngineChip: View {
+    @Environment(\.themePalette) private var palette: ThemePalette
+
+    var body: some View {
+        HStack(spacing: 4) {
+            Image(systemName: "wand.and.stars")
+            ExtL10n.text("keyboard.placeholder.cloudBadge")
+        }
+        .font(TypeStyle.caption2)
+        .foregroundStyle(palette.accent)
+        .padding(.horizontal, Spacing.xs + 2)
+        .padding(.vertical, 5)
+        .frame(minHeight: 26)
+        .background(palette.accent.opacity(0.15), in: Capsule())
+        .overlay(Capsule().stroke(palette.accent.opacity(0.35), lineWidth: 0.5))
+    }
+}
+
 // MARK: - Local engine chip (shown instead of ModeChip when engineMode == "local")

 private struct LocalEngineChip: View {
@@ -413,58 +459,6 @@ private struct LocalEngineChip: View {
    }
 }

-// MARK: - Mode chip
-
-private struct ModeChip: View {
-    @Environment(\.themePalette) private var palette: ThemePalette
-
-    let mode: KeyboardViewController.State.InputMode
-    let onChange: (KeyboardViewController.State.InputMode) -> Void
-
-    var body: some View {
-        Menu {
-            ForEach(KeyboardViewController.State.InputMode.allCases) { m in
-                Button {
-                    onChange(m)
-                } label: {
-                    if m == mode {
-                        Label(label(for: m), systemImage: "checkmark")
-                    } else {
-                        Text(label(for: m))
-                    }
-                }
-            }
-        } label: {
-            HStack(spacing: 4) {
-                Image(systemName: icon(for: mode))
-                Text(label(for: mode))
-                Image(systemName: "chevron.down")
-                    .font(.system(size: 8, weight: .bold))
-            }
-            .font(TypeStyle.caption2)
-            .foregroundStyle(mode == .off ? palette.textTertiary : palette.textPrimary)
-            .padding(.horizontal, Spacing.xs + 2)
-            .padding(.vertical, 5)
-            .frame(minHeight: 26)
-            .background(palette.surfaceElevated, in: Capsule())
-            .overlay(Capsule().stroke(palette.divider, lineWidth: 0.5))
-        }
-        .menuStyle(.button)
-    }
-
-    private func label(for m: KeyboardViewController.State.InputMode) -> String {
-        ExtL10n.string(m.labelKey)
-    }
-
-    private func icon(for m: KeyboardViewController.State.InputMode) -> String {
-        switch m {
-        case .off:        return "mic.slash.fill"
-        case .transcribe: return "text.bubble.fill"
-        case .polish:     return "wand.and.stars"
-        }
-    }
-}
-
 // MARK: - Locale chip

 private struct LocaleChip: View {
@@ -69,12 +69,12 @@
 "settings.reset.title" = "Reset all settings?";
 "settings.reset.message" = "API key, model, and base URL will be cleared.";
 "settings.reset.confirm" = "Reset all settings";
-"settings.engine.title" = "Engine";
+"settings.engine.title" = "Recognition method";
 "settings.engine.subtitle" = "Pick the recognition engine. Local engine does transcription only, no API key needed.";
-"settings.engine.local.title" = "On-device";
+"settings.engine.local.title" = "On-device recognition";
 "settings.engine.local.ios26" = "Always on-device, no network.";
 "settings.engine.local.legacy" = "On-device ASR. Transcription only, no polish.";
-"settings.engine.cloud.title" = "Cloud polish";
+"settings.engine.cloud.title" = "Cloud recognition & polish";
 "settings.engine.cloud.subtitle" = "ASR + LLM polish. API key required.";
 "settings.provider.title" = "Provider";
 "settings.provider.subtitle" = "Pick the LLM that polishes your dictation.";
@@ -118,6 +118,10 @@
 "keyboard.placeholder.processing" = "Processing";
 "keyboard.placeholder.error" = "Polishing failed";
 "keyboard.placeholder.localBadge" = "On-device";
+"keyboard.placeholder.cloudBadge" = "Cloud";
+"keyboard.models.notDownloaded" = "On-device models not downloaded";
+"keyboard.models.downloadHint" = "Open OSGKeyboard to download models";
+"keyboard.models.warming" = "Loading models…";
 "keyboard.rec" = "REC";
 "keyboard.space" = "Space";
 "keyboard.denied.mic" = "Mic denied";
@@ -69,12 +69,12 @@
 "settings.reset.title" = "重置所有设置?";
 "settings.reset.message" = "API key、model 和 base URL 都会被清空。";
 "settings.reset.confirm" = "重置所有设置";
-"settings.engine.title" = "引擎";
+"settings.engine.title" = "识别方式";
 "settings.engine.subtitle" = "选择识别方式。本地引擎无需 API Key,仅做语音转录。";
 "settings.engine.local.title" = "本地识别";
 "settings.engine.local.ios26" = "始终端侧,无需联网。";
 "settings.engine.local.legacy" = "端侧 ASR,仅转录,无润色。";
-"settings.engine.cloud.title" = "云端润色";
+"settings.engine.cloud.title" = "云端识别与润色";
 "settings.engine.cloud.subtitle" = "ASR 转录 + LLM 润色,需要 API Key。";
 "settings.provider.title" = "提供商";
 "settings.provider.subtitle" = "选择 LLM 提供商。";
@@ -118,6 +118,10 @@
 "keyboard.placeholder.processing" = "处理中…";
 "keyboard.placeholder.error" = "润色失败";
 "keyboard.placeholder.localBadge" = "本地";
+"keyboard.placeholder.cloudBadge" = "云端";
+"keyboard.models.notDownloaded" = "本地模型尚未下载";
+"keyboard.models.downloadHint" = "打开 OSGKeyboard 下载模型";
+"keyboard.models.warming" = "正在加载模型…";
 "keyboard.rec" = "REC";
 "keyboard.space" = "空格";
 "keyboard.denied.mic" = "麦克风被拒绝";