feat: migrate on-device Qwen3 ASR to CoreML for background Flow dictation

Replace MLX GPU inference with CoreML bundles so transcription continues while the host app is backgrounded. Adds model download and warm-up, vendored Qwen3Speech, and updates onboarding, settings, and copy for the ~1.6 GB CoreML package (iOS 18+).
2026-06-23 00:46:58 +08:00
parent 5e5122f172
commit df1c5ff32c
160 changed files with 22080 additions and 492 deletions
@@ -0,0 +1,64 @@
+// ProgressiveDictationTranscriptAccumulator.swift
+// OSGKeyboard · Shared
+//
+// Merges progressive `DictationTranscriber` results into one transcript.
+// Short-form presets may emit a new time range after ~30 s; treating the
+// latest partial as the full transcript drops earlier segments.
+
+import Foundation
+import CoreMedia
+
+/// Combines volatile partials and finalized segments from
+/// `DictationTranscriber.results` into a single growing transcript.
+public struct ProgressiveDictationTranscriptAccumulator: Sendable {
+
+    private struct Segment: Sendable {
+        let startSeconds: Double
+        var text: String
+    }
+
+    private var segments: [Segment] = []
+    private var lastEmitted = ""
+
+    public init() {}
+
+    /// Ingest one analyzer result. Returns a non-nil full transcript when the
+    /// composed text changed since the previous emission.
+    public mutating func ingest(range: CMTimeRange, text: String) -> String? {
+        let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
+        guard !trimmed.isEmpty else { return nil }
+
+        let start = range.start.seconds
+
+        if let idx = segments.lastIndex(where: { abs($0.startSeconds - start) < 0.001 }) {
+            // Same audio window — volatile refinement of the current segment.
+            segments[idx].text = trimmed
+        } else if let last = segments.last,
+                  trimmed.hasPrefix(last.text) || last.text.hasPrefix(trimmed) {
+            // Cumulative progressive update without a range change.
+            let longer = trimmed.count >= last.text.count ? trimmed : last.text
+            segments[segments.count - 1].text = longer
+        } else {
+            // New time range — append instead of replacing earlier speech.
+            segments.append(Segment(startSeconds: start, text: trimmed))
+        }
+
+        let full = composedText()
+        guard full != lastEmitted else { return nil }
+        lastEmitted = full
+        return full
+    }
+
+    /// Final composed transcript after the results stream finishes.
+    public mutating func finalize() -> String {
+        let full = composedText()
+        lastEmitted = full
+        return full
+    }
+
+    private func composedText() -> String {
+        segments.reduce(into: "") { partial, segment in
+            partial = DictationTextComposer.compose(anchor: partial, live: segment.text)
+        }
+    }
+}
@@ -6,9 +6,12 @@
 import Foundation

 public enum ProviderDisplayName {
-    public static func name(for providerId: String) -> String {
+    public static func name(
+        for providerId: String,
+        language: AppUILanguage? = nil
+    ) -> String {
        let key = "provider.\(providerId)"
-        let localized = NSLocalizedString(key, comment: "")
+        let localized = SharedL10n.string(key, language: language)
        if localized != key { return localized }
        return LLMProvider.provider(id: providerId).name
    }
@@ -0,0 +1,101 @@
+// UtteranceStreamChunker.swift
+// OSGKeyboard · Shared
+//
+// Splits a Flow utterance PCM stream into ASR-sized chunks. When possible,
+// extends slightly past the max window to the next pause instead of cutting
+// mid-word.
+
+import Foundation
+
+public enum UtteranceStreamChunker {
+
+    /// Yields chunks as audio arrives; the final chunk is marked `isLast`.
+    public static func chunks(
+        from stream: AsyncStream<AudioBufferSnapshot>,
+        config: FlowUtteranceChunkConfig = .flowDefault
+    ) -> AsyncStream<UtteranceAudioChunk> {
+        AsyncStream { continuation in
+            let task = Task {
+                var buffer: [Float] = []
+                buffer.reserveCapacity(config.maxChunkSamples + config.pauseExtensionSamples)
+                var chunkIndex = 0
+
+                func emit(upTo splitEnd: Int, isLast: Bool) {
+                    guard splitEnd > 0, splitEnd <= buffer.count else { return }
+                    let chunkSamples = Array(buffer[..<splitEnd])
+                    continuation.yield(
+                        UtteranceAudioChunk(index: chunkIndex, samples: chunkSamples, isLast: isLast)
+                    )
+                    chunkIndex += 1
+                    if splitEnd >= buffer.count {
+                        buffer.removeAll(keepingCapacity: true)
+                    } else {
+                        let overlapStart = max(0, splitEnd - config.overlapSamples)
+                        buffer = Array(buffer[overlapStart...])
+                    }
+                }
+
+                for await snap in stream {
+                    if Task.isCancelled { break }
+                    guard !snap.samples.isEmpty else { continue }
+                    buffer.append(contentsOf: snap.samples)
+
+                    while buffer.count >= config.maxChunkSamples {
+                        let split = pauseAwareSplitIndex(in: buffer, config: config)
+                        emit(upTo: split, isLast: false)
+                    }
+                }
+
+                if !buffer.isEmpty {
+                    emit(upTo: buffer.count, isLast: true)
+                } else if chunkIndex == 0 {
+                    // Empty utterance — no chunks.
+                } else {
+                    // Stream ended exactly on boundary; mark prior path complete.
+                }
+
+                continuation.finish()
+            }
+
+            continuation.onTermination = { _ in
+                task.cancel()
+            }
+        }
+    }
+
+    /// Pick a split index at or after `maxChunkSamples`, preferring a pause.
+    static func pauseAwareSplitIndex(
+        in buffer: [Float],
+        config: FlowUtteranceChunkConfig
+    ) -> Int {
+        let minSplit = config.maxChunkSamples
+        guard buffer.count >= minSplit else { return buffer.count }
+
+        let searchEnd = min(buffer.count, minSplit + config.pauseExtensionSamples)
+        if searchEnd <= minSplit {
+            return minSplit
+        }
+
+        let windowSize = max(config.sampleRate / 50, 160) // ~20 ms
+        var bestPause: Int?
+        var idx = minSplit
+        while idx + windowSize <= searchEnd {
+            if rms(of: buffer, start: idx, count: windowSize) < config.pauseRMSThreshold {
+                bestPause = idx + windowSize
+            }
+            idx += windowSize / 2
+        }
+
+        return bestPause ?? minSplit
+    }
+
+    static func rms(of samples: [Float], start: Int, count: Int) -> Float {
+        guard start >= 0, count > 0, start + count <= samples.count else { return 1 }
+        var sum: Float = 0
+        for i in start..<(start + count) {
+            let v = samples[i]
+            sum += v * v
+        }
+        return sqrtf(sum / Float(count))
+    }
+}
@@ -0,0 +1,109 @@
+// UtteranceTranscriptStitcher.swift
+// OSGKeyboard · Shared
+//
+// Orders pipelined chunk transcripts and merges overlap at boundaries.
+
+import Foundation
+
+public struct UtteranceTranscriptStitcher: Sendable {
+    private var segments: [(index: Int, text: String)] = []
+
+    public init() {}
+
+    public mutating func append(index: Int, text: String) {
+        let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
+        guard !trimmed.isEmpty else { return }
+        if let existing = segments.firstIndex(where: { $0.index == index }) {
+            segments[existing].text = trimmed
+        } else {
+            segments.append((index, trimmed))
+            segments.sort { $0.index < $1.index }
+        }
+    }
+
+    public func composed() -> String {
+        guard let first = segments.first else { return "" }
+        var result = first.text
+        for segment in segments.dropFirst() {
+            result = Self.mergeWithOverlap(previous: result, next: segment.text)
+        }
+        return result
+    }
+
+    /// Merge `next` onto `previous`, dropping duplicated suffix/prefix overlap.
+    public static func mergeWithOverlap(previous: String, next: String) -> String {
+        let trimmedNext = next.trimmingCharacters(in: .whitespacesAndNewlines)
+        guard !trimmedNext.isEmpty else { return previous }
+        guard !previous.isEmpty else { return trimmedNext }
+
+        // Character-granular probe — works for CJK without word boundaries.
+        let prevChars = Array(previous)
+        let nextChars = Array(trimmedNext)
+        let maxProbe = min(64, prevChars.count, nextChars.count)
+        if maxProbe > 0 {
+            for length in stride(from: maxProbe, through: 1, by: -1) {
+                let suffix = prevChars.suffix(length)
+                let prefix = nextChars.prefix(length)
+                if suffix.elementsEqual(prefix) {
+                    return previous + String(nextChars.dropFirst(length))
+                }
+            }
+        }
+
+        // Punctuation-insensitive CJK overlap (e.g. "很好，" + "很好继续").
+        let normalizedPrev = normalizeForOverlap(previous)
+        let normalizedNext = normalizeForOverlap(trimmedNext)
+        let nPrev = Array(normalizedPrev)
+        let nNext = Array(normalizedNext)
+        let normProbe = min(64, nPrev.count, nNext.count)
+        if normProbe > 0 {
+            for length in stride(from: normProbe, through: 2, by: -1) {
+                if nPrev.suffix(length).elementsEqual(nNext.prefix(length)) {
+                    // Map normalized overlap length back to raw `next` drop count.
+                    let drop = overlapDropCount(in: trimmedNext, normalizedPrefixLength: length)
+                    return previous + String(trimmedNext.dropFirst(drop))
+                }
+            }
+        }
+
+        // English / spaced languages.
+        let maxWordProbe = min(6, previous.split(separator: " ").count, trimmedNext.split(separator: " ").count)
+        if maxWordProbe > 0 {
+            let prevWords = previous.split(separator: " ", omittingEmptySubsequences: true)
+            let nextWords = trimmedNext.split(separator: " ", omittingEmptySubsequences: true)
+            for wordCount in stride(from: maxWordProbe, through: 1, by: -1) {
+                if prevWords.suffix(wordCount).elementsEqual(nextWords.prefix(wordCount)) {
+                    let mergedPrefix = nextWords.dropFirst(wordCount).joined(separator: " ")
+                    if mergedPrefix.isEmpty { return previous }
+                    if previous.last == " " || previous.last == "\n" {
+                        return previous + mergedPrefix
+                    }
+                    return previous + " " + mergedPrefix
+                }
+            }
+        }
+
+        return DictationTextComposer.compose(anchor: previous, live: trimmedNext)
+    }
+
+    private static func normalizeForOverlap(_ text: String) -> String {
+        text.unicodeScalars.filter {
+            !CharacterSet.whitespacesAndNewlines.contains($0)
+                && !CharacterSet.punctuationCharacters.contains($0)
+        }.map { Character($0) }.reduce(into: "") { $0.append($1) }
+    }
+
+    /// How many raw characters to drop from `next` given a normalized-prefix overlap length.
+    private static func overlapDropCount(in next: String, normalizedPrefixLength: Int) -> Int {
+        var normalizedCount = 0
+        var rawIndex = next.startIndex
+        while rawIndex < next.endIndex, normalizedCount < normalizedPrefixLength {
+            let scalar = next[rawIndex]
+            if !scalar.isWhitespace, !scalar.isPunctuation {
+                normalizedCount += 1
+            }
+            rawIndex = next.index(after: rawIndex)
+        }
+        return next.distance(from: next.startIndex, to: rawIndex)
+    }
+}