feat: migrate on-device Qwen3 ASR to CoreML for background Flow dictation

Replace MLX GPU inference with CoreML bundles so transcription continues while the host app is backgrounded. Adds model download and warm-up, vendored Qwen3Speech, and updates onboarding, settings, and copy for the ~1.6 GB CoreML package (iOS 18+).
2026-06-23 00:46:58 +08:00
parent 5e5122f172
commit df1c5ff32c
160 changed files with 22080 additions and 492 deletions
@@ -16,6 +16,7 @@

 import Foundation
 import AVFoundation
+import CoreMedia
 import Speech
 import os

@@ -42,6 +43,63 @@ public protocol ASRService: Sendable {

    /// Cancel any in-flight recognition and tear down its tasks.
    func cancel()
+
+    /// Clears cancellation / cached session state before a new utterance.
+    func resetForNewUtterance()
+
+    /// Transcribe one PCM chunk (Flow pipelined path). Default wraps `transcribe(stream:)`.
+    func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult
+}
+
+public enum ASRChunkResult: Sendable, Equatable {
+    case success(String)
+    case failure(String)
+    case cancelled
+}
+
+extension ASRService {
+    public func resetForNewUtterance() {}
+
+    public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
+        guard !samples.isEmpty else { return .success("") }
+        if Task.isCancelled { return .cancelled }
+
+        let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
+        let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
+        continuation.yield(snapshot)
+        continuation.finish()
+
+        var lastPartial = ""
+        var finalText = ""
+        var failure: String?
+
+        for await event in transcribe(stream: stream, locale: locale) {
+            if Task.isCancelled { return .cancelled }
+            switch event {
+            case .capability:
+                break
+            case .partial(let text):
+                lastPartial = text
+            case .final(let text):
+                finalText = text
+            case .error(let message):
+                failure = message
+            }
+        }
+
+        if let failure {
+            return .failure(failure)
+        }
+        let trimmed = finalText.trimmingCharacters(in: .whitespacesAndNewlines)
+        if !trimmed.isEmpty {
+            return .success(trimmed)
+        }
+        let partial = lastPartial.trimmingCharacters(in: .whitespacesAndNewlines)
+        if !partial.isEmpty {
+            return .success(partial)
+        }
+        return .success("")
+    }
 }

 public enum ASREvent: Sendable, Equatable {
@@ -58,13 +116,68 @@ public enum ASREvent: Sendable, Equatable {
 // MARK: - Factory

 public enum ASRServiceFactory {
-    /// Returns the ASR backend. With iOS 26 as the deployment target,
-    /// there is exactly one backend (`SpeechAnalyzer`).
+    /// Registry of backend-specific providers. The host app installs
+    /// a provider for the Qwen3-ASR backend at launch time (the
+    /// `Qwen3ASRProvider` lives in the app target because linking
+    /// `Qwen3ASR` pulls in mlx-swift, which the shared framework
+    /// deliberately stays off to keep `APPLICATION_EXTENSION_API_ONLY`
+    /// clean). The shared framework always provides a built-in
+    /// `SpeechAnalyzer` provider; custom providers override it.
+    ///
+    /// `nonisolated(unsafe)` because the only writer is
+    /// `OSGKeyboardApp.init` (single-threaded, runs once at launch).
+    /// After launch, all callers read the dictionary from any
+    /// actor.
+    public nonisolated(unsafe) static var providers: [LocalASRBackend: any ASRServiceProvider] = [
+        .speechAnalyzer: SpeechAnalyzerProvider()
+    ]
+
+    /// Returns the ASR backend chosen by the user. The cloud engine
+    /// always uses the iOS `SpeechAnalyzer` path — it has the lowest
+    /// latency and never hits the network, which matches the user's
+    /// expectation that "ASR" is the local half of the pipeline
+    /// regardless of where the LLM polish happens.
+    ///
+    /// For the local engine, we honour `LocalASRBackend`:
+    ///   - `.speechAnalyzer` (default) → on-device iOS pipeline.
+    ///   - `.qwen3ASR` → CoreML-backed Qwen3-ASR via `soniqo/speech-swift` (host app only)
+    ///                     (registered by the host app at launch).
+    public static func make(
+        engineMode: String,
+        localBackend: LocalASRBackend = .speechAnalyzer
+    ) -> ASRService {
+        if engineMode == "local" {
+            if let provider = providers[localBackend] {
+                return provider.make()
+            }
+        }
+        return SpeechAnalyzerASR()
+    }
+
+    /// Back-compat overload for callers that only ever want the
+    /// SpeechAnalyzer path. The previous single-backend build used
+    /// this signature; new code should pass the engine mode explicitly
+    /// so the user's selection is honoured.
    public static func make() -> ASRService {
        SpeechAnalyzerASR()
    }
 }

+/// Backend-specific ASR factory. The shared framework ships a default
+/// `SpeechAnalyzerProvider`; the host app installs a `Qwen3ASRProvider`
+/// at launch time so the Qwen3 backend is wired in only where its
+/// large MLX dependency is also linked.
+public protocol ASRServiceProvider: Sendable {
+    var backend: LocalASRBackend { get }
+    func make() -> ASRService
+}
+
+/// Built-in provider for the iOS SpeechAnalyzer path. Always present.
+struct SpeechAnalyzerProvider: ASRServiceProvider {
+    let backend: LocalASRBackend = .speechAnalyzer
+    func make() -> ASRService { SpeechAnalyzerASR() }
+}
+
 // MARK: - PCM format conversion (testable helpers)
 //
 // Extracted from the audio-thread hot path so the scaling + clipping
@@ -115,6 +228,114 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
    private var analyzer: SpeechAnalyzer?
    private var analyzerTask: Task<Void, Never>?
    private var analyzerFinished = false
+    /// Reused across pipelined chunks within one utterance (assets + format).
+    private var chunkPreparedLocaleID: String?
+    private var chunkAnalyzerFormat: AVAudioFormat?
+
+    func resetForNewUtterance() {
+        lock.withLock {
+            chunkPreparedLocaleID = nil
+            chunkAnalyzerFormat = nil
+        }
+    }
+
+    func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
+        guard !samples.isEmpty else { return .success("") }
+        if Task.isCancelled { return .cancelled }
+
+        do {
+            let text = try await transcribeSamples(samples, locale: locale, reuseChunkPrep: true)
+            let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
+            return trimmed.isEmpty ? .success("") : .success(trimmed)
+        } catch is CancellationError {
+            return .cancelled
+        } catch {
+            return .failure(error.localizedDescription)
+        }
+    }
+
+    /// Analyze a single PCM buffer without the streaming `transcribe` wrapper.
+    private func transcribeSamples(
+        _ samples: [Float],
+        locale: Locale,
+        reuseChunkPrep: Bool
+    ) async throws -> String {
+        guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
+            throw ASRChunkError.localeUnsupported
+        }
+        let localeID = resolvedLocale.identifier(.bcp47)
+        let transcriber = DictationTranscriber(
+            locale: resolvedLocale,
+            preset: .progressiveLongDictation
+        )
+
+        let analyzerFormat: AVAudioFormat
+        let cachedPrep = lock.withLock { (chunkPreparedLocaleID, chunkAnalyzerFormat) }
+        if reuseChunkPrep,
+           cachedPrep.0 == localeID,
+           let cached = cachedPrep.1 {
+            analyzerFormat = cached
+        } else {
+            try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
+            guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
+                compatibleWith: [transcriber],
+                considering: Self.captureFormat
+            ) else {
+                throw ASRChunkError.formatUnsupported
+            }
+            analyzerFormat = format
+            lock.withLock {
+                chunkPreparedLocaleID = localeID
+                chunkAnalyzerFormat = format
+            }
+        }
+
+        let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
+        guard let pcm = Self.makeAnalyzerPCMBuffer(from: snapshot, format: analyzerFormat) else {
+            throw ASRChunkError.formatUnsupported
+        }
+
+        let analyzer = SpeechAnalyzer(modules: [transcriber])
+        try await analyzer.prepareToAnalyze(in: analyzerFormat)
+
+        let resultsTask = Task<String, Error> {
+            var accumulator = ProgressiveDictationTranscriptAccumulator()
+            for try await result in transcriber.results {
+                if Task.isCancelled { break }
+                let text = String(result.text.characters)
+                _ = accumulator.ingest(range: result.range, text: text)
+            }
+            return accumulator.finalize()
+        }
+
+        let inputStream = AsyncStream<AnalyzerInput> { continuation in
+            continuation.yield(AnalyzerInput(buffer: pcm))
+            continuation.finish()
+        }
+
+        let lastSampleTime = try await analyzer.analyzeSequence(inputStream)
+        if let lastSampleTime {
+            try await analyzer.finalizeAndFinish(through: lastSampleTime)
+        } else {
+            await analyzer.cancelAndFinishNow()
+        }
+
+        return try await resultsTask.value
+    }
+
+    private enum ASRChunkError: LocalizedError {
+        case localeUnsupported
+        case formatUnsupported
+
+        var errorDescription: String? {
+            switch self {
+            case .localeUnsupported:
+                return SharedL10n.string("error.asr.localeUnsupported")
+            case .formatUnsupported:
+                return SharedL10n.string("error.asr.formatUnsupported")
+            }
+        }
+    }

    /// Canonical capture format: 16 kHz mono Float32 from `AudioCaptureService`
    /// / `PreviewASRController` before it reaches SpeechAnalyzer.
@@ -146,16 +367,21 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
                do {
                    guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
                        Self.debug("locale unsupported: \(locale.identifier(.bcp47))")
-                        continuation.yield(.error("当前系统未分配可用语音语言模型，请稍后重试或切换语言"))
+                        continuation.yield(.error(SharedL10n.string("error.asr.localeUnsupported")))
                        continuation.finish()
                        return
                    }
-                    let transcriber = DictationTranscriber(locale: resolvedLocale, preset: .progressiveShortDictation)
+                    // Each pipelined chunk is ≤ 30 s; long dictation preset keeps a
+                    // single chunk coherent (Flow utterances run up to 3 min).
+                    let transcriber = DictationTranscriber(
+                        locale: resolvedLocale,
+                        preset: .progressiveLongDictation
+                    )
                    do {
                        try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
                    } catch {
                        Self.debug("asset prepare failed: \(error.localizedDescription)")
-                        continuation.yield(.error("语音语言资源未就绪，请稍后重试"))
+                        continuation.yield(.error(SharedL10n.string("error.asr.assetsNotReady")))
                        continuation.finish()
                        return
                    }
@@ -167,7 +393,7 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
                        compatibleWith: [transcriber],
                        considering: Self.captureFormat
                    ) else {
-                        continuation.yield(.error("当前设备不支持该语音输入格式"))
+                        continuation.yield(.error(SharedL10n.string("error.asr.formatUnsupported")))
                        continuation.finish()
                        return
                    }
@@ -179,15 +405,16 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
                    // Apple recommends consuming `transcriber.results` concurrently
                    // while `analyzeSequence` drains the input stream.
                    let resultsTask = Task<String, Error> {
-                        var lastText = ""
+                        var accumulator = ProgressiveDictationTranscriptAccumulator()
                        for try await result in transcriber.results {
                            if Task.isCancelled { break }
                            let text = String(result.text.characters)
-                            guard !text.isEmpty, text != lastText else { continue }
-                            lastText = text
-                            continuation.yield(.partial(text))
+                            guard let full = accumulator.ingest(range: result.range, text: text) else {
+                                continue
+                            }
+                            continuation.yield(.partial(full))
                        }
-                        return lastText
+                        return accumulator.finalize()
                    }

                    let lastSampleTime = try await newAnalyzer.analyzeSequence(inputStream)
@@ -195,7 +422,7 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
                    if let lastSampleTime {
                        try await newAnalyzer.finalizeAndFinish(through: lastSampleTime)
                    } else {
-                        try await newAnalyzer.cancelAndFinishNow()
+                        await newAnalyzer.cancelAndFinishNow()
                    }

                    let lastText: String
@@ -210,7 +437,7 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {

                    let trimmed = lastText.trimmingCharacters(in: .whitespacesAndNewlines)
                    if trimmed.isEmpty {
-                        continuation.yield(.error("未识别到语音内容，请重试"))
+                        continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
                    } else {
                        continuation.yield(.final(trimmed))
                    }
@@ -35,6 +35,8 @@ public struct AppGroupStore: @unchecked Sendable {
        static let modeId          = "config.modeId"
        static let localeId        = "config.localeId"
        static let engineMode       = "config.engineMode"
+        static let localASRBackend  = "config.localASRBackend"
+        static let uiLanguage       = "config.uiLanguage"
    }

    // MARK: - Reads
@@ -70,12 +72,25 @@ public struct AppGroupStore: @unchecked Sendable {
        defaults.string(forKey: Key.localeId) ?? "auto"
    }

-    /// "local" → on-device ASR only, no LLM polishing.
+    /// "local" → on-device ASR only (raw transcript delivery).
    /// "cloud" → ASR + LLM polish (default behaviour).
    public var engineMode: String {
        defaults.string(forKey: Key.engineMode) ?? "cloud"
    }

+    /// Which on-device ASR engine backs the "local" engine mode. Falls
+    /// back to the iOS SpeechAnalyzer path so legacy installs (which
+    /// never wrote this key) keep working.
+    public var localASRBackend: LocalASRBackend {
+        let raw = defaults.string(forKey: Key.localASRBackend) ?? LocalASRBackend.speechAnalyzer.rawValue
+        return LocalASRBackend(rawValue: raw) ?? .speechAnalyzer
+    }
+
+    /// Host-app UI language override (`auto` / `en` / `zh-Hans`).
+    public var uiLanguage: AppUILanguage {
+        AppUILanguage.fromStored(defaults.string(forKey: Key.uiLanguage))
+    }
+
    // MARK: - Writes

    public func setModeId(_ id: String) {
@@ -90,6 +105,14 @@ public struct AppGroupStore: @unchecked Sendable {
        defaults.set(mode, forKey: Key.engineMode)
    }

+    public func setLocalASRBackend(_ backend: LocalASRBackend) {
+        defaults.set(backend.rawValue, forKey: Key.localASRBackend)
+    }
+
+    public func setUILanguage(_ language: AppUILanguage) {
+        defaults.set(language.rawValue, forKey: Key.uiLanguage)
+    }
+
    // MARK: - Client

    public func makeClient() -> LLMClient {
@@ -0,0 +1,160 @@
+// ChunkedUtterancePipeline.swift
+// OSGKeyboard · Shared
+//
+// Pipelined Flow utterance ASR: split PCM while recording, transcribe chunks
+// serially on a background queue, stitch partials for display and delivery.
+
+import Foundation
+
+public struct ChunkedUtteranceSuccess: Sendable, Equatable {
+    public let text: String
+    /// Non-fatal per-chunk ASR issues (delivered as soft warning when non-empty).
+    public let chunkWarnings: [String]
+
+    public init(text: String, chunkWarnings: [String] = []) {
+        self.text = text
+        self.chunkWarnings = chunkWarnings
+    }
+}
+
+public enum ChunkedUtterancePipelineOutcome: Sendable, Equatable {
+    case success(ChunkedUtteranceSuccess)
+    case failure(String)
+    case cancelled
+}
+
+/// Thread-safe queue between the chunk feeder and ASR worker.
+private actor ChunkWorkQueue {
+    private var items: [UtteranceAudioChunk] = []
+    private var finished = false
+    private var waiters: [CheckedContinuation<UtteranceAudioChunk?, Never>] = []
+
+    func enqueue(_ chunk: UtteranceAudioChunk) {
+        items.append(chunk)
+        resumeWaiters()
+    }
+
+    func markFinished() {
+        finished = true
+        resumeWaiters()
+    }
+
+    func dequeue() async -> UtteranceAudioChunk? {
+        if !items.isEmpty {
+            return items.removeFirst()
+        }
+        if finished {
+            return nil
+        }
+        return await withCheckedContinuation { continuation in
+            waiters.append(continuation)
+        }
+    }
+
+    private func resumeWaiters() {
+        while !waiters.isEmpty {
+            if !items.isEmpty {
+                let waiter = waiters.removeFirst()
+                waiter.resume(returning: items.removeFirst())
+            } else if finished {
+                let waiter = waiters.removeFirst()
+                waiter.resume(returning: nil)
+            } else {
+                break
+            }
+        }
+    }
+}
+
+public actor ChunkedUtterancePipeline {
+    private let asr: ASRService
+    private let locale: Locale
+    private let config: FlowUtteranceChunkConfig
+    private var cancelled = false
+
+    public init(
+        asr: ASRService,
+        locale: Locale,
+        config: FlowUtteranceChunkConfig = .flowDefault
+    ) {
+        self.asr = asr
+        self.locale = locale
+        self.config = config
+    }
+
+    public func cancel() {
+        cancelled = true
+        asr.cancel()
+    }
+
+    /// Consume `stream` until finished; ASR runs off the caller's actor while recording continues.
+    public func transcribe(
+        stream: AsyncStream<AudioBufferSnapshot>,
+        onPartial: @Sendable @escaping (String) -> Void
+    ) async -> ChunkedUtterancePipelineOutcome {
+        asr.resetForNewUtterance()
+
+        let queue = ChunkWorkQueue()
+        var stitcher = UtteranceTranscriptStitcher()
+        var chunkWarnings: [String] = []
+        var failedChunks = 0
+        var processedChunks = 0
+
+        let feeder = Task {
+            for await chunk in UtteranceStreamChunker.chunks(from: stream, config: config) {
+                if Task.isCancelled { break }
+                await queue.enqueue(chunk)
+            }
+            await queue.markFinished()
+        }
+
+        while true {
+            if cancelled || Task.isCancelled {
+                feeder.cancel()
+                return .cancelled
+            }
+
+            guard let chunk = await queue.dequeue() else { break }
+
+            processedChunks += 1
+            let asr = self.asr
+            let locale = self.locale
+            let result = await Task.detached(priority: .userInitiated) {
+                await asr.transcribeChunk(samples: chunk.samples, locale: locale)
+            }.value
+
+            switch result {
+            case .success(let text):
+                stitcher.append(index: chunk.index, text: text)
+                let partial = stitcher.composed()
+                if !partial.isEmpty {
+                    onPartial(partial)
+                }
+            case .failure(let message):
+                failedChunks += 1
+                chunkWarnings.append(
+                    SharedL10n.format(
+                        "error.asr.chunkFailed",
+                        chunk.index + 1,
+                        message
+                    )
+                )
+            case .cancelled:
+                feeder.cancel()
+                return .cancelled
+            }
+        }
+
+        _ = await feeder.value
+
+        let finalText = stitcher.composed().trimmingCharacters(in: .whitespacesAndNewlines)
+        if finalText.isEmpty {
+            if failedChunks > 0, processedChunks == failedChunks {
+                return .failure(SharedL10n.string("error.asr.noSpeech"))
+            }
+            return .failure(SharedL10n.string("error.asr.noSpeech"))
+        }
+
+        return .success(ChunkedUtteranceSuccess(text: finalText, chunkWarnings: chunkWarnings))
+    }
+}
@@ -21,6 +21,7 @@ public enum DictationBridge {

    private enum Key {
        static let pendingText = "dictation.pendingText"
+        static let polishWarning = "dictation.polishWarning"
        static let updatedAt = "dictation.updatedAt"
        static let status = "dictation.status"
        static let statusUpdatedAt = "dictation.statusUpdatedAt"
@@ -67,12 +68,21 @@ public enum DictationBridge {
    }

    /// Store a transcript for the keyboard extension to consume.
-    public static func storePendingTranscript(_ text: String, defaults: UserDefaults? = nil) {
+    public static func storePendingTranscript(
+        _ text: String,
+        polishWarning: String? = nil,
+        defaults: UserDefaults? = nil
+    ) {
        let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
        guard !trimmed.isEmpty else { return }
        let store = resolvedDefaults(defaults)
        store.set(trimmed, forKey: Key.pendingText)
        store.set(Date().timeIntervalSince1970, forKey: Key.updatedAt)
+        if let polishWarning, !polishWarning.isEmpty {
+            store.set(polishWarning, forKey: Key.polishWarning)
+        } else {
+            store.removeObject(forKey: Key.polishWarning)
+        }
        setStatus(.done, defaults: store)
    }

@@ -81,6 +91,15 @@ public enum DictationBridge {
        maxAge: TimeInterval = 180,
        defaults: UserDefaults? = nil
    ) -> String? {
+        consumePendingDelivery(maxAge: maxAge, defaults: defaults)?.text
+    }
+
+    /// Returns and clears the pending delivery (text + optional polish
+    /// warning) if present.
+    public static func consumePendingDelivery(
+        maxAge: TimeInterval = 180,
+        defaults: UserDefaults? = nil
+    ) -> TranscriptionDelivery? {
        let store = resolvedDefaults(defaults)
        guard let text = store.string(forKey: Key.pendingText) else {
            return nil
@@ -92,14 +111,18 @@ public enum DictationBridge {
                return nil
            }
        }
+        let warning = store.string(forKey: Key.polishWarning)
        store.removeObject(forKey: Key.pendingText)
+        store.removeObject(forKey: Key.polishWarning)
+        store.removeObject(forKey: Key.updatedAt)
        setStatus(.idle, defaults: store)
-        return text
+        return TranscriptionDelivery(text: text, polishWarning: warning)
    }

    public static func clear(defaults: UserDefaults? = nil) {
        let store = resolvedDefaults(defaults)
        store.removeObject(forKey: Key.pendingText)
+        store.removeObject(forKey: Key.polishWarning)
        store.removeObject(forKey: Key.updatedAt)
        setStatus(.idle, defaults: store)
    }
@@ -0,0 +1,39 @@
+// FlowAppLifecycle.swift
+// OSGKeyboard · Shared
+//
+// Tracks whether the host app process is in the foreground.
+// Retained for any future GPU-backed paths; CoreML ASR does not require it.
+
+import Foundation
+
+public final class FlowAppLifecycle: @unchecked Sendable {
+
+    public static let shared = FlowAppLifecycle()
+
+    private let lock = NSLock()
+    private var isForeground = true
+
+    private init() {}
+
+    /// `true` when the host app scene is active (`.active`).
+    public var allowsGPUInference: Bool {
+        lock.lock()
+        defer { lock.unlock() }
+        return isForeground
+    }
+
+    public func setForeground(_ foreground: Bool) {
+        lock.lock()
+        isForeground = foreground
+        lock.unlock()
+    }
+
+    /// Blocks until foreground or cancellation.
+    public func waitUntilForeground() async -> Bool {
+        while !allowsGPUInference {
+            if Task.isCancelled { return false }
+            try? await Task.sleep(nanoseconds: 200_000_000)
+        }
+        return true
+    }
+}
@@ -34,7 +34,7 @@ private final class FlowCaptureStreamRelay: @unchecked Sendable {
    }

    func yield(_ snapshot: AudioBufferSnapshot) {
-        lock.withLock { continuation?.yield(snapshot) }
+        _ = lock.withLock { continuation?.yield(snapshot) }
    }

    func finish() {
@@ -250,6 +250,22 @@ public final class FlowContinuousCapture {
        )
    }

+    /// Re-activate capture after returning from background without
+    /// reinstalling the tap (iOS may deactivate the audio session).
+    public func reassertIfRunning() {
+        guard isRunning else { return }
+        let session = AVAudioSession.sharedInstance()
+        try? session.setCategory(
+            .playAndRecord,
+            mode: .measurement,
+            options: [.defaultToSpeaker, .allowBluetoothHFP, .mixWithOthers]
+        )
+        try? session.setActive(true, options: .notifyOthersOnDeactivation)
+        if !audioEngine.isRunning {
+            try? audioEngine.start()
+        }
+    }
+
    /// Begin forwarding downsampled buffers to ASR for one utterance.
    public func beginUtterance() -> AsyncStream<AudioBufferSnapshot> {
        let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
@@ -65,12 +65,24 @@ public enum FlowSessionBridge {

    // MARK: - Session validity (keyboard)

-    /// True when expires is in the future and heartbeat is fresh.
+    /// True when the session contract is still valid (not expired).
+    /// Does not require a fresh heartbeat — the host may be suspended in
+    /// background while the continuous audio session is frozen.
    public static func isSessionActive(defaults: UserDefaults? = nil) -> Bool {
        let store = resolvedDefaults(defaults)
        flush(store)
+        guard store.bool(forKey: FlowSessionKeys.flowSessionActive) else { return false }
+
        let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires)
-        guard expires > Date().timeIntervalSince1970 else { return false }
+        return expires > Date().timeIntervalSince1970
+    }
+
+    /// True when the host app recently wrote a heartbeat (foreground or
+    /// actively processing). Used for auto-start heuristics, not gating record.
+    public static func isHostReachable(defaults: UserDefaults? = nil) -> Bool {
+        let store = resolvedDefaults(defaults)
+        flush(store)
+        guard isSessionActive(defaults: store) else { return false }

        let heartbeat = store.double(forKey: FlowSessionKeys.flowHeartbeat)
        guard heartbeat > 0 else { return false }
@@ -124,6 +136,7 @@ public enum FlowSessionBridge {

    public static func storeTranscriptionResult(
        _ text: String,
+        polishWarning: String? = nil,
        defaults: UserDefaults? = nil
    ) {
        let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -131,6 +144,11 @@ public enum FlowSessionBridge {
        let store = resolvedDefaults(defaults)
        store.set(trimmed, forKey: FlowSessionKeys.transcriptionResult)
        store.removeObject(forKey: FlowSessionKeys.transcriptionError)
+        if let polishWarning, !polishWarning.isEmpty {
+            store.set(polishWarning, forKey: FlowSessionKeys.transcriptionPolishWarning)
+        } else {
+            store.removeObject(forKey: FlowSessionKeys.transcriptionPolishWarning)
+        }
        setRecordingState(.idle, defaults: store)
        flush(store)
    }
@@ -147,14 +165,24 @@ public enum FlowSessionBridge {

    /// Returns and clears a pending transcription result, if any.
    public static func consumeTranscriptionResult(defaults: UserDefaults? = nil) -> String? {
+        consumeTranscriptionDelivery(defaults: defaults)?.text
+    }
+
+    /// Returns and clears a pending transcription delivery (text + optional
+    /// polish warning), if any.
+    public static func consumeTranscriptionDelivery(
+        defaults: UserDefaults? = nil
+    ) -> TranscriptionDelivery? {
        let store = resolvedDefaults(defaults)
        flush(store)
        guard let text = store.string(forKey: FlowSessionKeys.transcriptionResult), !text.isEmpty else {
            return nil
        }
+        let warning = store.string(forKey: FlowSessionKeys.transcriptionPolishWarning)
        store.removeObject(forKey: FlowSessionKeys.transcriptionResult)
+        store.removeObject(forKey: FlowSessionKeys.transcriptionPolishWarning)
        flush(store)
-        return text
+        return TranscriptionDelivery(text: text, polishWarning: warning)
    }

    /// Returns and clears a pending transcription error, if any.
@@ -212,6 +240,7 @@ public enum FlowSessionBridge {

    private static func clearTranscription(defaults: UserDefaults) {
        defaults.removeObject(forKey: FlowSessionKeys.transcriptionResult)
+        defaults.removeObject(forKey: FlowSessionKeys.transcriptionPolishWarning)
        defaults.removeObject(forKey: FlowSessionKeys.transcriptionError)
    }
 }
@@ -13,17 +13,39 @@ public enum FlowSessionKeys {
    public static let keyboardRecordingState = "flow.keyboardRecordingState"
    public static let transcriptionLanguage = "flow.transcriptionLanguage"
    public static let transcriptionResult = "flow.transcriptionResult"
+    /// Soft warning when polish failed but raw transcript was delivered.
+    public static let transcriptionPolishWarning = "flow.transcriptionPolishWarning"
    public static let transcriptionError = "flow.transcriptionError"
    public static let audioLevels = "flow.audioLevels"

-    /// Heartbeat older than this implies the host app was killed.
+    /// Heartbeat older than this while the host is foreground → likely killed.
    public static let heartbeatStaleInterval: TimeInterval = 3

    /// Default Flow session length when started from the keyboard.
    public static let defaultSessionDuration: TimeInterval = 480

-    /// Maximum duration for a single keyboard utterance.
-    public static let maxUtteranceDuration: TimeInterval = 60
+    /// Maximum duration for a single keyboard utterance (3 minutes).
+    public static let maxUtteranceDuration: TimeInterval = 180
+
+    /// Host polls for pipelined ASR drain after mic stop. Pipelining usually
+    /// finishes most chunks during recording; this is a soft deadline before
+    /// blocking on `asrTask.value` (which waits until the pipeline exits).
+    public static let localASRWaitTimeout: TimeInterval = 120
+    public static let localQwen3ASRWaitTimeout: TimeInterval = 180
+    public static let cloudASRWaitTimeout: TimeInterval = 120
+
+    /// Keyboard watchdog after the user stops recording (not utterance max length).
+    /// Must cover worst-case post-stop backlog: remaining MLX/SpeechAnalyzer chunks
+    /// plus cloud LLM polish (see `PolishingService.effectiveTimeout` cap).
+    public static func keyboardResultTimeout(
+        engineMode: String,
+        localASRBackend: LocalASRBackend
+    ) -> TimeInterval {
+        if engineMode == "local" {
+            return localASRBackend == .qwen3ASR ? 240 : 180
+        }
+        return 240
+    }

    public enum RecordingState: String, Sendable, Equatable {
        case idle
@@ -71,8 +71,17 @@ public final class KeyboardState: ObservableObject {
    @Published public var utteranceRemainingSeconds: Int = Int(FlowSessionKeys.maxUtteranceDuration)
    /// Whether the host app's Flow voice session is currently valid.
    @Published public var flowSessionActive: Bool = false
-    /// "local" → ASR only, no LLM. "cloud" → ASR + optional LLM polish.
+    /// "local" → on-device ASR only. "cloud" → ASR + LLM polish.
    @Published public var engineMode: String = "cloud"
+    /// Which on-device ASR engine to use when `engineMode == "local"`.
+    /// Mirrored from `ProviderConfig.localASRBackend` for UI display
+    /// and for `state` consumers that want a single source of truth.
+    @Published public var localASRBackend: LocalASRBackend = .speechAnalyzer
+    /// `false` when the local engine needs on-device models that are
+    /// not yet downloaded (mirrored from App Group by the extension).
+    @Published public var localModelsReady: Bool = true
+    /// `true` when host app has preloaded Qwen weights into memory.
+    @Published public var localModelsLoaded: Bool = false

    /// Convenience shorthand used by the pipeline and views.
    public var isLocalEngine: Bool { engineMode == "local" }
@@ -86,6 +95,7 @@ public final class KeyboardState: ObservableObject {
    public var setMode:             (InputMode) -> Void = { _ in }
    public var setLocale:           (String) -> Void = { _ in }
    public var setEngineMode:        (String) -> Void = { _ in }
+    public var setLocalASRBackend:  (LocalASRBackend) -> Void = { _ in }
    public var insertNewline:       () -> Void = {}
    public var insertSpace:         () -> Void = {}
    public var deleteBackward:      () -> Void = {}
@@ -17,13 +17,20 @@ public enum LLMError: Error, LocalizedError, Sendable, Equatable {

    public var errorDescription: String? {
        switch self {
-        case .invalidURL:    return "API 地址无效。请在设置中检查 Base URL。"
-        case .noAPIKey:      return "未填写 API Key。"
-        case .http(let s):   return "API 返回 HTTP \(s)。请稍后重试或联系服务方。"
-        case .decoding:      return "解析 API 响应失败。"
-        case .transport:     return "网络错误,请检查连接后重试。"
-        case .rateLimited:   return "API 调用过于频繁,请稍候再试。"
-        case .cancelled:     return "请求已取消。"
+        case .invalidURL:
+            return SharedL10n.string("error.llm.invalidURL")
+        case .noAPIKey:
+            return SharedL10n.string("error.llm.noAPIKey")
+        case .http(let status):
+            return SharedL10n.format("error.llm.http", status)
+        case .decoding:
+            return SharedL10n.string("error.llm.decoding")
+        case .transport:
+            return SharedL10n.string("error.llm.transport")
+        case .rateLimited:
+            return SharedL10n.string("error.llm.rateLimited")
+        case .cancelled:
+            return SharedL10n.string("error.llm.cancelled")
        }
    }
 }
@@ -36,7 +36,7 @@ private final class CaptureStreamRelay: @unchecked Sendable {
    }

    func yield(_ snapshot: AudioBufferSnapshot) {
-        lock.withLock { continuation?.yield(snapshot) }
+        _ = lock.withLock { continuation?.yield(snapshot) }
    }

    func finish() {
@@ -73,7 +73,7 @@ public final class LiveDictationController: ObservableObject {
    /// next recording starts from zero.
    @Published public var lastFinal: String = ""

-    private let asr: ASRService = ASRServiceFactory.make()
+    private let asr: ASRService
    private let audioEngine = AVAudioEngine()
    /// `internal` (not `private`) so the regression test in
    /// `OSGKeyboardTests/PreviewASRControllerStateTests.swift` can
@@ -83,10 +83,20 @@ public final class LiveDictationController: ObservableObject {
    /// code outside the class from racing on it.
    public var asrTask: Task<Void, Never>?
    private let streamRelay = CaptureStreamRelay()
+    private var chunkedPipeline: ChunkedUtterancePipeline?
    private var didConfigureAudioSession = false
    private var didInstallTap = false

-    public init() {}
+    public init(asr: ASRService? = nil) {
+        // Resolve through the factory so the user's `LocalASRBackend`
+        // selection is honoured. Tests can pass a stub `asr` directly
+        // to bypass the factory and exercise the controller in
+        // isolation.
+        self.asr = asr ?? ASRServiceFactory.make(
+            engineMode: ProviderConfig.shared.engineMode,
+            localBackend: ProviderConfig.shared.localASRBackend
+        )
+    }

    /// Start dictation using a persisted settings locale id (`auto`, `zh-Hans`, …).
    public func start(localeId: String) async {
@@ -112,6 +122,10 @@ public final class LiveDictationController: ObservableObject {
        // same `events` stream.
        asrTask?.cancel()
        asrTask = nil
+        if let pipeline = chunkedPipeline {
+            Task { await pipeline.cancel() }
+        }
+        chunkedPipeline = nil
        teardownCapturePipeline()
        phase = .requestingPermission
        currentPartial = ""
@@ -335,29 +349,36 @@ public final class LiveDictationController: ObservableObject {
            return
        }

-        // 5. Wire up ASR.
-        let events = asr.transcribe(
-            stream: stream,
-            locale: locale
-        )
-        asrTask = Task { @MainActor [weak self] in
-            guard let self else { return }
-            for await event in events {
-                switch event {
-                case .capability:
-                    break
-                case .partial(let s):
-                    self.currentPartial = s
-                case .final(let s):
-                    let trimmed = s.trimmingCharacters(in: .whitespacesAndNewlines)
-                    self.lastFinal = trimmed
-                    self.currentPartial = ""
-                    self.phase = .idle
-                case .error(let m):
-                    self.debug("asr error: \(m)")
-                    self.teardownCapturePipeline()
-                    self.errorMessage = m
-                    self.phase = .error(m)
+        // 5. Pipelined ASR (same chunk path as Flow host).
+        let pipeline = ChunkedUtterancePipeline(asr: asr, locale: locale)
+        chunkedPipeline = pipeline
+        asrTask = Task.detached(priority: .userInitiated) { [weak controller = self] in
+            let outcome = await pipeline.transcribe(stream: stream) { partial in
+                Task { @MainActor in
+                    controller?.currentPartial = partial
+                }
+            }
+            await MainActor.run {
+                guard let controller else { return }
+                switch outcome {
+                case .success(let success):
+                    let trimmed = success.text.trimmingCharacters(in: .whitespacesAndNewlines)
+                    if !trimmed.isEmpty {
+                        controller.lastFinal = trimmed
+                        controller.currentPartial = ""
+                    }
+                    if controller.phase == .processing || controller.phase == .recording {
+                        controller.phase = .idle
+                    }
+                case .failure(let message):
+                    controller.debug("asr error: \(message)")
+                    controller.teardownCapturePipeline()
+                    controller.errorMessage = message
+                    controller.phase = .error(message)
+                case .cancelled:
+                    if controller.phase == .processing {
+                        controller.phase = .idle
+                    }
                }
            }
        }
@@ -0,0 +1,104 @@
+// OnDeviceModelStatus.swift
+// OSGKeyboard · Shared
+//
+// Mirrors on-device model download state into the App Group so the
+// keyboard extension can show readiness hints without reading the
+// host app's Caches directory.
+
+import Foundation
+
+public enum OnDeviceModelStatus {
+
+    private enum Key {
+        static func downloaded(_ model: OnDeviceModel) -> String {
+            "models.\(model.rawValue).downloaded"
+        }
+        static func progress(_ model: OnDeviceModel) -> String {
+            "models.\(model.rawValue).downloadProgress"
+        }
+        static let modelsLoadedInMemory = "models.loadedInMemory"
+    }
+
+    // MARK: - Writes (host app)
+
+    public static func setDownloaded(_ downloaded: Bool, for model: OnDeviceModel) {
+        guard AppGroup.isAvailable else { return }
+        AppGroup.defaults.set(downloaded, forKey: Key.downloaded(model))
+        if downloaded {
+            clearProgress(for: model)
+        }
+    }
+
+    public static func setProgress(_ progress: Double?, for model: OnDeviceModel) {
+        guard AppGroup.isAvailable else { return }
+        if let progress {
+            AppGroup.defaults.set(progress, forKey: Key.progress(model))
+        } else {
+            AppGroup.defaults.removeObject(forKey: Key.progress(model))
+        }
+    }
+
+    public static func clearProgress(for model: OnDeviceModel) {
+        guard AppGroup.isAvailable else { return }
+        AppGroup.defaults.removeObject(forKey: Key.progress(model))
+    }
+
+    public static func setModelsLoadedInMemory(_ loaded: Bool) {
+        guard AppGroup.isAvailable else { return }
+        AppGroup.defaults.set(loaded, forKey: Key.modelsLoadedInMemory)
+    }
+
+    public static func modelsLoadedInMemory(defaults: UserDefaults? = nil) -> Bool {
+        let store = defaults ?? (AppGroup.isAvailable ? AppGroup.defaults : .standard)
+        return store.bool(forKey: Key.modelsLoadedInMemory)
+    }
+
+    // MARK: - Reads (keyboard + host app)
+
+    public static func isDownloaded(_ model: OnDeviceModel, defaults: UserDefaults? = nil) -> Bool {
+        let store = defaults ?? (AppGroup.isAvailable ? AppGroup.defaults : .standard)
+        return store.bool(forKey: Key.downloaded(model))
+    }
+
+    public static func downloadProgress(_ model: OnDeviceModel, defaults: UserDefaults? = nil) -> Double? {
+        let store = defaults ?? (AppGroup.isAvailable ? AppGroup.defaults : .standard)
+        guard store.object(forKey: Key.progress(model)) != nil else { return nil }
+        return store.double(forKey: Key.progress(model))
+    }
+
+    /// Whether the currently selected local-engine stack has every
+    /// required on-device model downloaded.
+    public static func isLocalStackReady(
+        asrBackend: LocalASRBackend,
+        defaults: UserDefaults? = nil
+    ) -> Bool {
+        if asrBackend == .qwen3ASR {
+            return isDownloaded(.qwen3ASR, defaults: defaults)
+        }
+        return true
+    }
+
+    /// First missing model for the active local stack, if any.
+    public static func firstMissingModel(
+        asrBackend: LocalASRBackend,
+        defaults: UserDefaults? = nil
+    ) -> OnDeviceModel? {
+        if asrBackend == .qwen3ASR, !isDownloaded(.qwen3ASR, defaults: defaults) {
+            return .qwen3ASR
+        }
+        return nil
+    }
+}
+
+// MARK: - On-device Qwen3 runtime
+
+/// CoreML ASR requires iOS 18+ / macOS 15+ (MLState KV cache).
+public enum OnDeviceMLRuntime {
+    /// Whether Qwen3-ASR CoreML can run in this process.
+    public static var supportsOnDeviceQwen3: Bool {
+        if #available(iOS 18.0, *) {
+            return true
+        }
+        return false
+    }
+}
@@ -5,9 +5,9 @@
 // to produce polished, well-punctuated text. Falls back to the raw transcript
 // if the LLM call fails or times out.
 //
-// Mode-aware: when `modeId == "off"` the service short-circuits and returns
-// the trimmed input without touching the network. This is the runtime
-// guarantee behind the keyboard's "Off · 关闭" mode.
+// Cloud engine always runs the LLM polish step (settings no longer expose
+// off / transcribe). Local engine (`engineMode == "local"`) is ASR-only —
+// the raw transcript is returned unchanged and cloud API settings are ignored.

 import Foundation

@@ -16,7 +16,6 @@ public actor PolishingService {
    public enum PolishError: Error, Equatable {
        case noTranscript
        case timeout
-        case modeOff
    }

    private let store: AppGroupStore
@@ -44,25 +43,25 @@ public actor PolishingService {
        let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
        guard !trimmed.isEmpty else { throw PolishError.noTranscript }

-        // Mode-aware short-circuit. When the user has selected "Off", the
-        // keyboard must never hit the network — we return the trimmed
-        // input as-is. This is the same value the view controller would
-        // produce if it skipped `polish()` entirely, but having the
-        // guarantee at the service layer means future call sites (CLI,
-        // tests, alternate keyboards) inherit it for free.
-        if store.modeId == "off" {
+        // Local engine: ASR-only — no on-device or cloud polish.
+        if store.engineMode == "local" {
            return trimmed
        }

+        return try await polishRemote(trimmed)
+    }
+
+    private func polishRemote(_ trimmed: String) async throws -> String {
        let client = injectedClient ?? store.makeClient()
        let prompt = store.systemPrompt
+        let budget = effectiveTimeout(for: trimmed)

        return try await withThrowingTaskGroup(of: String.self) { group in
            group.addTask {
                try await client.polish(trimmed, systemPrompt: prompt)
            }
            group.addTask {
-                try await Task.sleep(nanoseconds: UInt64(self.timeout * 1_000_000_000))
+                try await Task.sleep(nanoseconds: UInt64(budget * 1_000_000_000))
                throw PolishError.timeout
            }
            let result = try await group.next()!
@@ -70,4 +69,10 @@ public actor PolishingService {
            return result
        }
    }
-}
+
+    /// Scale polish budget with transcript length (3-minute Flow utterances).
+    private func effectiveTimeout(for text: String) -> TimeInterval {
+        let scaled = timeout + (Double(text.count) / 200.0) * 2.0
+        return min(max(scaled, timeout), 120)
+    }
+}