From c0c9dad14936a0d4a81b1c38abf55f892aea28bd Mon Sep 17 00:00:00 2001 From: Rocky <72559939+hkgood@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:34:56 +0800 Subject: [PATCH] feat(mac): add Qwen3 MLX streaming dictation Replace the Sherpa offline pipeline with native MLX streaming, resilient model downloads, live transcript previews, and supporting tests and documentation. --- .gitignore | 3 + CHANGELOG.md | 5 + .../Services/OpenSourceLicenseCatalog.swift | 38 ++- OSGKeyboardMac/DashboardView.swift | 88 +++--- .../MacDictationOverlayController.swift | 145 +++++++++- OSGKeyboardMac/MacDictationOverlayView.swift | 43 ++- OSGKeyboardMac/MacDictationPipeline.swift | 121 ++------- OSGKeyboardMac/MacDictationViewModel.swift | 70 ++++- OSGKeyboardMac/MacHallucinationFilter.swift | 85 ++++++ OSGKeyboardMac/MacLocalASRChunkAdapter.swift | 43 --- .../MacLocalASRModelSettingsView.swift | 48 +++- OSGKeyboardMac/MacLocalASRService.swift | 60 +++-- OSGKeyboardMac/MacMLXLiveCapture.swift | 150 +++++++++++ .../MacMLXStreamingASRProvider.swift | 91 +++++++ OSGKeyboardMac/MacMLXStreamingSession.swift | 86 ++++++ OSGKeyboardMac/MacSherpaLocalASR.swift | 64 ----- OSGKeyboardMac/MacSherpaONNXRunner.swift | 253 ------------------ .../Models/LocalASRCapabilities.swift | 4 +- .../Models/LocalASRModelCatalog.swift | 63 ++++- .../Resources/LocalASR/local-asr-catalog.json | 175 +++++------- .../LocalASRModelDownloadClient.swift | 78 +++++- .../Services/LocalASRModelInstallState.swift | 5 + .../Services/LocalASRModelManager.swift | 27 +- .../Services/LocalASRPreferenceKeys.swift | 2 + OSGKeyboardShared/en.lproj/Shared.strings | 7 +- .../zh-Hans.lproj/Shared.strings | 7 +- .../LocalASRDownloadSourceSorterTests.swift | 25 +- .../LocalASRModelCatalogTests.swift | 104 ++++--- Scripts/ensure-mlx-audio-swift.sh | 68 +++++ Scripts/generate-xcodeproj.sh | 4 + Scripts/patch-spm-local-package.sh | 60 +++++ ThirdParty/README.md | 15 ++ docs/local-asr-architecture.md | 2 +- .../mac-qwen3-mlx-streaming-migration-plan.md | 84 ++++-- project.yml | 14 +- 35 files changed, 1373 insertions(+), 764 deletions(-) create mode 100644 OSGKeyboardMac/MacHallucinationFilter.swift delete mode 100644 OSGKeyboardMac/MacLocalASRChunkAdapter.swift create mode 100644 OSGKeyboardMac/MacMLXLiveCapture.swift create mode 100644 OSGKeyboardMac/MacMLXStreamingASRProvider.swift create mode 100644 OSGKeyboardMac/MacMLXStreamingSession.swift delete mode 100644 OSGKeyboardMac/MacSherpaLocalASR.swift delete mode 100644 OSGKeyboardMac/MacSherpaONNXRunner.swift create mode 100755 Scripts/ensure-mlx-audio-swift.sh create mode 100755 Scripts/patch-spm-local-package.sh create mode 100644 ThirdParty/README.md diff --git a/.gitignore b/.gitignore index a5f5db1..801a04a 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,9 @@ OSGKeyboard/Resources/CustomLanguageModel/v1/compiled/ __pycache__/ *.pyc +# Vendored mlx-audio-swift (fetched by Scripts/ensure-mlx-audio-swift.sh) +ThirdParty/mlx-audio-swift/ + # Local debug capture logs and screenshots .flow-capture.log .flow-oslog-capture.log diff --git a/CHANGELOG.md b/CHANGELOG.md index c536fbf..4b764e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- **Mac MLX streaming ASR**: local dictation uses Qwen3-ASR via mlx-audio-swift with overlay partial preview, tail drain, vocabulary prompt, and polish-before-insert. / **Mac MLX 流式 ASR**:本地听写改用 mlx-audio-swift 的 Qwen3-ASR,支持浮层 partial 预览、尾部截断、词库 prompt 与润色后再插入。 + +### Changed +- **Mac local ASR engine**: removed Sherpa offline CLI; default model is Qwen3 MLX 0.6B 4-bit (1.7B optional download). / **Mac 本地 ASR 引擎**:移除 Sherpa offline CLI;默认模型改为 Qwen3 MLX 0.6B 4-bit(1.7B 可选下载)。 - **Landing competitor matrix**: compare Typeless / Superwhisper / Openless / OSGKeyboard on open source, pricing, on-device ASR, BYOK, and platforms (incl. honest Windows gap). / **落地页竞品对照**:对比 Typeless / Superwhisper / Openless / OSGKeyboard 的开源、付费、本地识别、BYOK 与平台(含暂无 Windows)。 - **Voluntary support tip**: Settings (top of the page) includes an optional ¥28 Consumable in-app tip (StoreKit 2). All features stay free — no paywall or unlock. / **自愿打赏**:设置页顶部新增可选 ¥28 消耗型应用内打赏(StoreKit 2)。全功能仍免费,无付费墙或功能解锁。 - **iOS appearance preference**: Settings → Preferences adds System / Light / Dark (iPhone + iPad), matching the Mac control. / **iOS 外观偏好**:设置 → 偏好设置新增跟随系统 / 浅色 / 深色(iPhone 与 iPad),与 Mac 一致。 @@ -21,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Landing hero polish**: replace CSS device frames with the marketing composite; full-bleed pale-green hero wash (no side gaps / no radial gradient); Mac story shots sit on transparent chrome. / **落地页 Hero 抛光**:设备框改为营销合成图;首屏淡绿单色通栏(无两侧留白 / 无径向渐变);Mac 故事截图去卡片底。 ### Fixed +- **Model download for China networks**: add the `hf-mirror.com` mirror as the mainland-China-first source (bypasses the flaky HF Xet backend), drop the dead ModelScope link, and correct the model file list; add automatic resume-on-drop retry with cross-mirror fallback and a manual "Download source" picker in Settings. / **国内网络模型下载**:新增 `hf-mirror.com` 镜像并在中国大陆优先(绕开不稳定的 HF Xet 后端),删除失效的 ModelScope 链接并修正模型文件清单;下载中断自动断点续传重试并在镜像间回退,设置里新增手动「下载源」选择。 - **iPad sidebar brand mark**: use the template `OSGLogoWide` mark with accent tint so the logo stays visible in the split-view sidebar. / **iPad 侧栏品牌标**:改用可着色的 `OSGLogoWide`,保证分栏侧栏始终显示 logo。 ## [0.5.3] - 2026-07-11 diff --git a/OSGKeyboard/Services/OpenSourceLicenseCatalog.swift b/OSGKeyboard/Services/OpenSourceLicenseCatalog.swift index a1e9558..33eca3f 100644 --- a/OSGKeyboard/Services/OpenSourceLicenseCatalog.swift +++ b/OSGKeyboard/Services/OpenSourceLicenseCatalog.swift @@ -6,8 +6,8 @@ // // Keep this list aligned with bundled resources and runtime downloads. // -// iOS targets remain zero-SPM. macOS local ASR downloads the sherpa-onnx -// runtime binary on demand; model weights are cached under Application Support. +// iOS targets remain zero-SPM. macOS local ASR links mlx-audio-swift (vendored +// under ThirdParty/) for Qwen3 MLX streaming; model weights download via catalog. import Foundation @@ -35,12 +35,12 @@ enum OpenSourceLicenseCatalog { licenseText: apache2Text ), .init( - id: "sherpa-onnx", - name: "sherpa-onnx", - licenseName: "Apache-2.0", - purpose: "macOS local ASR runtime (`sherpa-onnx-offline`) downloaded at install time and cached on device.", - url: URL(string: "https://github.com/k2-fsa/sherpa-onnx"), - licenseText: apache2Text + id: "mlx-audio-swift", + name: "mlx-audio-swift", + licenseName: "MIT", + purpose: "macOS local Qwen3 MLX streaming ASR (MLXAudioSTT), linked only in the Mac app target.", + url: URL(string: "https://github.com/Blaizzy/mlx-audio-swift"), + licenseText: mitText ), ] @@ -63,4 +63,26 @@ enum OpenSourceLicenseCatalog { implied. See the License for the specific language governing permissions and limitations under the License. """ + + static let mitText = """ + MIT License + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + """ } diff --git a/OSGKeyboardMac/DashboardView.swift b/OSGKeyboardMac/DashboardView.swift index 117871d..eb3f239 100644 --- a/OSGKeyboardMac/DashboardView.swift +++ b/OSGKeyboardMac/DashboardView.swift @@ -22,33 +22,35 @@ struct DashboardView: View { } var body: some View { - // Scrollable like the other pages so the shared status footer always - // stays pinned to the window's bottom edge. `minHeight: viewport` keeps - // the balanced Spacer layout when the window is tall (no scrollbar) and - // lets the content scroll only when the window is too short to fit it. + // 布局意图:标题 + 统计卡片顶对齐,麦克风栏底对齐,中间的听写预览卡片 + // 吸收全部剩余高度(弹性),因此拉高窗口时是卡片变高而非上下留白。 + // + // 不用 ScrollView:ScrollView 在测量内容时给子视图的是「理想高度」提案, + // 会让 `.frame(maxHeight: .infinity)` 的卡片停在最小高度、多余空间变留白。 + // 主窗口最小高度 600,内容恒可容纳,故直接用 GeometryReader 定高铺满 + // (与 iOS 端 HomeView 的弹性预览框同款、已验证可行的模式)。 GeometryReader { proxy in - ScrollView { - VStack(spacing: 0) { - VStack(alignment: .leading, spacing: Spacing.lg) { - heroHeader - statCluster - dictationStage - } - .padding(.horizontal, MacMetrics.pageHorizontalInset) - .padding(.top, Spacing.sm) - - // Leftover window height splits evenly above / below the mic - // bar so spacing stays balanced at any window size. The top - // gap keeps a larger floor so the mic never hugs the canvas. - Spacer(minLength: 30) - - BottomDictationBar(viewModel: viewModel) - .padding(.horizontal, MacMetrics.pageHorizontalInset) - - Spacer(minLength: Spacing.xs) + VStack(spacing: 0) { + // 顶部固定区块:品牌标题 + 统计卡片。 + VStack(alignment: .leading, spacing: Spacing.lg) { + heroHeader + statCluster } - .frame(maxWidth: .infinity, minHeight: proxy.size.height) + .padding(.horizontal, MacMetrics.pageHorizontalInset) + .padding(.top, Spacing.sm) + + // 中间弹性区块:预览卡片撑满标题与麦克风之间的剩余空间。 + dictationStage + .padding(.horizontal, MacMetrics.pageHorizontalInset) + .padding(.vertical, Spacing.lg) + .frame(maxWidth: .infinity, maxHeight: .infinity) + + // 底部固定区块:麦克风按钮 + 两侧信息。 + BottomDictationBar(viewModel: viewModel) + .padding(.horizontal, MacMetrics.pageHorizontalInset) + .padding(.bottom, Spacing.xs) } + .frame(width: proxy.size.width, height: proxy.size.height, alignment: .top) } .onAppear { stats.reloadFromDisk() } .onReceive(NotificationCenter.default.publisher(for: .usageStatisticsDidSyncFromCloud)) { _ in @@ -105,7 +107,24 @@ struct DashboardView: View { private var dictationStage: some View { MacCard(padding: Spacing.md, cornerRadius: Radius.large) { ZStack(alignment: .topLeading) { - if viewModel.transcript.isEmpty { + if viewModel.hasHomePreview { + // 总览:累积所有会话文本,内部滚动、自动贴底显示最新一句。 + ScrollView { + Text(viewModel.homePreviewText) + .font(.system(size: 20, weight: .regular)) + .foregroundStyle(palette.textPrimary) + .lineSpacing(4) + .textSelection(.enabled) + .frame(maxWidth: .infinity, alignment: .topLeading) + } + .defaultScrollAnchor(.bottom) + .frame( + maxWidth: .infinity, + minHeight: MacMetrics.dictationCanvasMinHeight, + maxHeight: .infinity, + alignment: .topLeading + ) + } else { Text( viewModel.isRecording ? MacL10n.string("mac.status.listening", language: lang) @@ -121,24 +140,13 @@ struct DashboardView: View { alignment: .topLeading ) .transition(.opacity) - } else { - Text(viewModel.transcript) - .font(.system(size: 20, weight: .regular)) - .foregroundStyle(palette.textPrimary) - .lineSpacing(4) - .textSelection(.enabled) - .frame( - maxWidth: .infinity, - minHeight: MacMetrics.dictationCanvasMinHeight, - maxHeight: .infinity, - alignment: .topLeading - ) - .transition(.opacity) } } - .frame(minHeight: MacMetrics.dictationCanvasMinHeight, maxHeight: 160) + // 不再封顶:卡片跟随外层弹性容器把剩余窗口高度全部吃满。 + .frame(minHeight: MacMetrics.dictationCanvasMinHeight, maxHeight: .infinity) } - .animation(Motion.soft, value: viewModel.transcript.isEmpty) + .frame(maxHeight: .infinity) + .animation(Motion.soft, value: viewModel.hasHomePreview) .animation(Motion.quick, value: viewModel.isRecording) } diff --git a/OSGKeyboardMac/MacDictationOverlayController.swift b/OSGKeyboardMac/MacDictationOverlayController.swift index 8cd895d..8ca9a18 100644 --- a/OSGKeyboardMac/MacDictationOverlayController.swift +++ b/OSGKeyboardMac/MacDictationOverlayController.swift @@ -24,7 +24,34 @@ final class MacDictationOverlayController { private let bottomMargin: CGFloat = 36 private let fallbackSize = NSSize(width: 400, height: 52) - private init() {} + // MARK: - User-draggable position (persisted across launches) + + /// True once the user has dragged the HUD; suppresses the default + /// bottom-center snap so the panel stays where the user placed it. + private var hasCustomPosition = false + /// Stored as center-X + bottom-left Y so the anchor stays stable while the + /// pill grows / shrinks with the live transcript (symmetric resize). + private var customCenterX: CGFloat = 0 + private var customOriginY: CGFloat = 0 + /// The origin we last set programmatically (kept for clamping / bookkeeping). + private var lastProgrammaticOrigin: NSPoint? + /// Cursor + window origin captured at the start of a manual drag, so we can + /// follow the absolute cursor and stay immune to the window moving under it. + private var dragCursorStart: NSPoint? + private var dragWindowStart: NSPoint? + + private static let hasCustomPositionKey = "mac.overlay.hasCustomPosition" + private static let centerXKey = "mac.overlay.centerX" + private static let originYKey = "mac.overlay.originY" + + private init() { + let defaults = UserDefaults.standard + if defaults.bool(forKey: Self.hasCustomPositionKey) { + hasCustomPosition = true + customCenterX = CGFloat(defaults.double(forKey: Self.centerXKey)) + customOriginY = CGFloat(defaults.double(forKey: Self.originYKey)) + } + } func start(observing viewModel: MacDictationViewModel) { guard cancellables.isEmpty else { return } @@ -134,6 +161,8 @@ final class MacDictationOverlayController { // full-screen apps, without going as high as the screen saver. panel.level = NSWindow.Level(rawValue: Int(CGWindowLevelForKey(.floatingWindow)) + 1) panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary] + // Background dragging can't move a non-activating panel; we drive the + // drag ourselves from a SwiftUI DragGesture (see `dragMoved`). panel.isMovableByWindowBackground = false panel.hidesOnDeactivate = false panel.ignoresMouseEvents = false @@ -147,10 +176,15 @@ final class MacDictationOverlayController { private func makeRoot(viewModel: MacDictationViewModel) -> AnyView { AnyView( - MacDictationOverlayView(viewModel: viewModel) - .macSystemPalette() - .environment(\.locale, viewModel.config.uiLanguage.swiftUILocale) - .preferredColorScheme(MacAppearancePreference.current.colorScheme) + MacDictationOverlayView( + viewModel: viewModel, + onDragChanged: { [weak self] in self?.dragMoved() }, + onDragEnded: { [weak self] in self?.dragEnded() }, + onResetPosition: { [weak self] in self?.resetPositionToDefault() } + ) + .macSystemPalette() + .environment(\.locale, viewModel.config.uiLanguage.swiftUILocale) + .preferredColorScheme(MacAppearancePreference.current.colorScheme) ) } @@ -158,18 +192,27 @@ final class MacDictationOverlayController { guard let panel, let hosting else { return } hosting.layoutSubtreeIfNeeded() let fitting = hosting.fittingSize + // Bounds include the 32pt horizontal transparent margin around the pill + // (16 per side) that gives the shadow room, so the pill body itself + // still spans ~300–520. let width = fitting.width.isFinite && fitting.width > 1 - ? min(max(fitting.width, 300), 520) + ? min(max(fitting.width, 332), 552) : fallbackSize.width let height = fitting.height.isFinite && fitting.height > 1 ? max(fitting.height, fallbackSize.height) : fallbackSize.height var frame = panel.frame - let midX = frame.midX + // Grow / shrink around the anchor center so the pill stays put: the + // dragged center when custom, otherwise its current center. + let targetMidX = hasCustomPosition ? customCenterX : frame.midX frame.size = NSSize(width: width, height: height) - if midX.isFinite { - frame.origin.x = midX - width / 2 + if targetMidX.isFinite { + frame.origin.x = targetMidX - width / 2 } + if let visible = (NSScreen.main ?? NSScreen.screens.first)?.visibleFrame { + frame.origin = clampedOrigin(frame.origin, size: frame.size, in: visible) + } + lastProgrammaticOrigin = frame.origin panel.setFrame(frame, display: true) hosting.frame = NSRect(origin: .zero, size: frame.size) } @@ -179,13 +222,89 @@ final class MacDictationOverlayController { let screen = NSScreen.main ?? NSScreen.screens.first guard let visible = screen?.visibleFrame else { return } let size = panel.frame.size - let origin = NSPoint( - x: visible.midX - size.width / 2, - y: visible.minY + bottomMargin - ) + // Respect the user's dragged spot; otherwise snap to bottom-center. + let desired: NSPoint + if hasCustomPosition { + desired = NSPoint(x: customCenterX - size.width / 2, y: customOriginY) + } else { + desired = NSPoint( + x: visible.midX - size.width / 2, + y: visible.minY + bottomMargin + ) + } + let origin = clampedOrigin(desired, size: size, in: visible) + lastProgrammaticOrigin = origin panel.setFrameOrigin(origin) } + /// Keep the panel fully inside the screen's visible frame so a dragged / + /// restored position can never strand it off-screen (e.g. after a display + /// or resolution change). + private func clampedOrigin(_ origin: NSPoint, size: NSSize, in visible: NSRect) -> NSPoint { + guard visible.width >= size.width, visible.height >= size.height else { + return origin + } + let x = min(max(origin.x, visible.minX), visible.maxX - size.width) + let y = min(max(origin.y, visible.minY), visible.maxY - size.height) + return NSPoint(x: x, y: y) + } + + /// Follows the absolute cursor while dragging. Reading `NSEvent.mouseLocation` + /// (screen coordinates) instead of the gesture's local translation avoids the + /// feedback loop you'd get from moving the window the gesture lives in. + private func dragMoved() { + guard let panel else { return } + let cursor = NSEvent.mouseLocation + if dragCursorStart == nil { + dragCursorStart = cursor + dragWindowStart = panel.frame.origin + } + guard let cursorStart = dragCursorStart, let windowStart = dragWindowStart else { return } + let target = NSPoint( + x: windowStart.x + (cursor.x - cursorStart.x), + y: windowStart.y + (cursor.y - cursorStart.y) + ) + let size = panel.frame.size + let visible = (NSScreen.main ?? NSScreen.screens.first)?.visibleFrame + let origin = visible.map { clampedOrigin(target, size: size, in: $0) } ?? target + lastProgrammaticOrigin = origin + panel.setFrameOrigin(origin) + } + + /// Persist the dragged spot as center-X + bottom-left Y. + private func dragEnded() { + dragCursorStart = nil + dragWindowStart = nil + guard let panel else { return } + customCenterX = panel.frame.midX + customOriginY = panel.frame.origin.y + hasCustomPosition = true + persistPosition() + } + + /// Double-clicking the pill clears the custom spot and returns it to the + /// default bottom-center. + private func resetPositionToDefault() { + hasCustomPosition = false + clearPersistedPosition() + resizeToFit() + reposition() + } + + private func persistPosition() { + let defaults = UserDefaults.standard + defaults.set(hasCustomPosition, forKey: Self.hasCustomPositionKey) + defaults.set(Double(customCenterX), forKey: Self.centerXKey) + defaults.set(Double(customOriginY), forKey: Self.originYKey) + } + + private func clearPersistedPosition() { + let defaults = UserDefaults.standard + defaults.removeObject(forKey: Self.hasCustomPositionKey) + defaults.removeObject(forKey: Self.centerXKey) + defaults.removeObject(forKey: Self.originYKey) + } + private func scheduleHide() { hideWorkItem?.cancel() let work = DispatchWorkItem { [weak self] in diff --git a/OSGKeyboardMac/MacDictationOverlayView.swift b/OSGKeyboardMac/MacDictationOverlayView.swift index bc24c40..a44e65f 100644 --- a/OSGKeyboardMac/MacDictationOverlayView.swift +++ b/OSGKeyboardMac/MacDictationOverlayView.swift @@ -9,6 +9,12 @@ import SwiftUI struct MacDictationOverlayView: View { @ObservedObject var viewModel: MacDictationViewModel + /// Called continuously while the user drags the pill (reads the live cursor + /// position on the controller side). Double-click resets to the default. + var onDragChanged: (() -> Void)? + var onDragEnded: (() -> Void)? + /// Double-click anywhere on the pill to snap it back to the default spot. + var onResetPosition: (() -> Void)? @Environment(\.themePalette) private var palette private var lang: AppUILanguage { viewModel.config.uiLanguage } @@ -35,6 +41,9 @@ struct MacDictationOverlayView: View { Spacer(minLength: Spacing.xs) trailingControl } + // Fixed content height so the pill never changes height between + // 识别中 (waveform, 28pt) and 润色中 (small spinner) states. + .frame(height: 28) .padding(.horizontal, Spacing.md) .padding(.vertical, 11) .frame(minWidth: 300, idealWidth: 400, maxWidth: 520) @@ -45,7 +54,22 @@ struct MacDictationOverlayView: View { .stroke(palette.dividerStrong, lineWidth: 0.5) ) .shadow(color: Color.black.opacity(0.22), radius: 14, y: 5) - .padding(2) + // Transparent margin large enough to contain the shadow's reach + // (radius 14 + y 5). The panel is sized to `fittingSize`, which ignores + // shadow, so without this room the borderless window clips the shadow + // into hard translucent-black corners. + .padding(EdgeInsets(top: 12, leading: 16, bottom: 20, trailing: 16)) + .contentShape(Capsule(style: .continuous)) + // Manual drag: `isMovableByWindowBackground` doesn't work on a + // non-activating panel, so we move the panel ourselves. The controller + // reads the live cursor position, so the translation value is unused. + .gesture( + DragGesture(minimumDistance: 3) + .onChanged { _ in onDragChanged?() } + .onEnded { _ in onDragEnded?() } + ) + .onTapGesture(count: 2) { onResetPosition?() } + .help(MacL10n.string("mac.overlay.dragHint", language: lang)) .animation(Motion.soft, value: hasPreview) .animation(Motion.quick, value: viewModel.isRecording) .animation(Motion.quick, value: viewModel.isStreamingPartial) @@ -65,9 +89,7 @@ struct MacDictationOverlayView: View { .foregroundStyle(palette.textPrimary) .lineLimit(1) .truncationMode(.head) - .frame(maxWidth: showsLiveBadge ? 280 : 320, alignment: .leading) - .contentTransition(.opacity) - .animation(Motion.quick, value: previewText) + .frame(maxWidth: showsLiveBadge ? 280 : 320, alignment: .trailing) .accessibilityLabel(previewText) } } else { @@ -133,8 +155,17 @@ struct MacDictationOverlayView: View { return MacL10n.string("mac.overlay.done", language: lang) } - @ViewBuilder + // Fixed-size trailing slot so the pill width / height stays steady as the + // control swaps between waveform, spinner and checkmark. private var trailingControl: some View { + HStack(spacing: Spacing.sm) { + trailingContent + } + .frame(height: 28, alignment: .trailing) + } + + @ViewBuilder + private var trailingContent: some View { if viewModel.isRecording { MiniWaveform( level: viewModel.audioLevel, @@ -149,10 +180,12 @@ struct MacDictationOverlayView: View { } else if viewModel.isPreparingToRecord || viewModel.isProcessing { ProgressView() .controlSize(.small) + .frame(width: 28, height: 28) } else { Image(systemName: "checkmark.circle.fill") .foregroundStyle(palette.accent) .symbolRenderingMode(.hierarchical) + .frame(width: 28, height: 28) } } diff --git a/OSGKeyboardMac/MacDictationPipeline.swift b/OSGKeyboardMac/MacDictationPipeline.swift index 7f64f04..f228b82 100644 --- a/OSGKeyboardMac/MacDictationPipeline.swift +++ b/OSGKeyboardMac/MacDictationPipeline.swift @@ -1,7 +1,7 @@ // MacDictationPipeline.swift // OSGKeyboard · Mac // -// Dictation pipeline: samples → ASR (cloud or local, chunked when long) → polish. +// Dictation pipeline: samples → ASR (cloud or local MLX streaming) → polish. import Foundation @@ -32,14 +32,11 @@ struct MacLiveASRCaptureResult: Sendable { } enum MacDictationPipeline { - /// First-chunk threshold: longer local utterances use pipelined chunk ASR. - private static let chunkedLocalThresholdSamples = Int( - FlowUtteranceChunkConfig.flowDefault.maxChunkDurationSeconds(forChunkIndex: 0) * 16_000 - ) - /// Whether the active engine can surface `onPartial` text while recording. static func supportsLivePartials(store: AppGroupStore) -> Bool { - if store.engineMode == "local" { return true } + if store.engineMode == "local" { + return MacLocalASRService.usesMLXLiveStreaming() + } let strategy = CloudASRModelCatalog.strategy(for: store.asrProviderId) return strategy != .localFallback } @@ -53,28 +50,17 @@ enum MacDictationPipeline { guard !samples.isEmpty else { throw MacDictationError.noAudio } let locale = resolvedLocale(store: store) - var chunkWarning: String? let raw: String var localBias: LocalASRBiasPayload? if store.engineMode == "local" { localBias = resolveLocalBias(store: store, locale: locale) - if samples.count > chunkedLocalThresholdSamples { - let chunked = try await transcribeLocalChunked( - samples: samples, - locale: locale, - bias: localBias, - onPartial: onPartial - ) - raw = chunked.text - chunkWarning = chunked.chunkWarning - } else { - raw = try await MacLocalASRService.transcribe( - samples: samples, - locale: locale, - bias: localBias - ) - } + raw = try await MacLocalASRService.transcribe( + samples: samples, + locale: locale, + bias: localBias + ) + onPartial?(raw) } else { let strategy = CloudASRModelCatalog.strategy(for: store.asrProviderId) guard strategy != .localFallback else { throw MacDictationError.providerHasNoCloudASR } @@ -93,16 +79,26 @@ enum MacDictationPipeline { raw: raw, store: store, localBias: localBias, - chunkWarning: chunkWarning + chunkWarning: nil ) } /// Consumes a live mic snapshot stream until finished; yields stitched partials. static func captureLive( stream: AsyncStream, + finishSignal: AsyncStream, store: AppGroupStore, onPartial: @escaping @Sendable (String) -> Void ) async -> MacLiveASRCaptureResult { + if store.engineMode == "local", MacLocalASRService.usesMLXLiveStreaming() { + return await MacMLXLiveCapture.run( + audioStream: stream, + finishSignal: finishSignal, + store: store, + onPartial: onPartial + ) + } + let locale = resolvedLocale(store: store) let localBias: LocalASRBiasPayload? if store.engineMode == "local" { @@ -112,11 +108,7 @@ enum MacDictationPipeline { } do { - let adapter = try makeChunkASRAdapter( - store: store, - locale: locale, - bias: localBias - ) + let adapter = try makeChunkASRAdapter(store: store) if let cloudAdapter = adapter as? MacCloudASRChunkAdapter { try? await cloudAdapter.prepare() } @@ -219,12 +211,7 @@ enum MacDictationPipeline { } } - // MARK: - Chunked local ASR - - private struct ChunkedLocalResult { - let text: String - let chunkWarning: String? - } + // MARK: - Private private static func resolvedLocale(store: AppGroupStore) -> Locale { Locale(identifier: store.localeId.isEmpty ? "zh-CN" : store.localeId) @@ -252,65 +239,7 @@ enum MacDictationPipeline { return bias } - private static func makeChunkASRAdapter( - store: AppGroupStore, - locale: Locale, - bias: LocalASRBiasPayload? - ) throws -> any ASRChunkTranscribing { - if store.engineMode == "local" { - return MacLocalASRChunkAdapter(locale: locale, bias: bias) - } - return try MacCloudASRChunkAdapter(store: store) - } - - private static func transcribeLocalChunked( - samples: [Float], - locale: Locale, - bias: LocalASRBiasPayload?, - onPartial: (@Sendable (String) -> Void)? - ) async throws -> ChunkedLocalResult { - let adapter = MacLocalASRChunkAdapter(locale: locale, bias: bias) - let pipeline = ChunkedUtterancePipeline( - asr: adapter, - locale: locale, - config: .flowDefault - ) - let outcome = await pipeline.transcribe( - stream: audioStream(from: samples), - onPartial: { partial in - onPartial?(partial) - } - ) - - switch outcome { - case .success(let success): - let warning = success.chunkWarnings.first - return ChunkedLocalResult(text: success.text, chunkWarning: warning) - case .failure(let message): - throw MacLocalASRError.qwen3InferenceFailed(message) - case .cancelled: - throw MacLocalASRError.qwen3InferenceFailed("Cancelled") - } - } - - /// Feeds recorded PCM into the chunker as if it arrived incrementally. - private static func audioStream( - from samples: [Float], - sliceSamples: Int = 8_000 - ) -> AsyncStream { - AsyncStream { continuation in - var offset = 0 - while offset < samples.count { - let end = min(offset + sliceSamples, samples.count) - continuation.yield( - AudioBufferSnapshot( - samples: Array(samples[offset.. any ASRChunkTranscribing { + try MacCloudASRChunkAdapter(store: store) } } diff --git a/OSGKeyboardMac/MacDictationViewModel.swift b/OSGKeyboardMac/MacDictationViewModel.swift index e7cf11f..0770747 100644 --- a/OSGKeyboardMac/MacDictationViewModel.swift +++ b/OSGKeyboardMac/MacDictationViewModel.swift @@ -52,7 +52,12 @@ final class MacDictationViewModel: ObservableObject { @Published var isProcessing = false /// True once live ASR has surfaced at least one partial during this take. @Published private(set) var isStreamingPartial = false + /// Live text for the *current* take (drives the floating HUD). Reset on + /// every new Option press. @Published var transcript = "" + /// Running overview of every finalized take this app run — the Home card + /// accumulates sessions here so earlier utterances are never overwritten. + @Published private(set) var overviewTranscript = "" @Published var statusMessage = "" @Published var audioLevel: Float = 0 @Published var sessionSeconds: Int = 0 @@ -74,9 +79,10 @@ final class MacDictationViewModel: ObservableObject { /// In-flight `beginRecording` started by the hotkey — cancelled if the /// key is released before the engine is ready (avoids a stuck session). private var hotkeyBeginTask: Task? - /// Live chunked ASR while recording (cloud / supported local paths). + /// Live chunked / streaming ASR while recording (cloud or MLX local). /// Finished in `finishRecording` so partials can become the final draft. private var liveCaptureTask: Task? + private var liveFinishContinuation: AsyncStream.Continuation? let usageStatistics: UsageStatisticsStore let speechHistory = SpeechHistoryStore.shared @@ -154,6 +160,22 @@ final class MacDictationViewModel: ObservableObject { transcript.split { $0 == " " || $0 == "\n" || $0 == "\t" }.count } + /// Text the Home overview card shows: all finalized takes plus the live + /// current take appended at the end while recording / processing. + var homePreviewText: String { + let live = transcript.trimmingCharacters(in: .whitespacesAndNewlines) + if live.isEmpty { return overviewTranscript } + if overviewTranscript.isEmpty { return live } + return overviewTranscript + "\n" + live + } + + var hasHomePreview: Bool { !homePreviewText.isEmpty } + + /// Clears the Home overview (the running session log), leaving any live take. + func clearOverview() { + overviewTranscript = "" + } + var isCloudMode: Bool { config.engineMode == "cloud" } var languageLabel: String { @@ -276,8 +298,19 @@ final class MacDictationViewModel: ObservableObject { ) stopTimers() audioLevel = 0 - let samples = recorder.stop() let store = AppGroupStore(defaults: defaults) + let usesDeferredStop = MacDictationPipeline.supportsLivePartials(store: store) + && store.engineMode == "local" + && MacLocalASRService.usesMLXLiveStreaming() + let samples: [Float] + if usesDeferredStop { + liveFinishContinuation?.yield(()) + samples = [] + } else { + liveFinishContinuation?.finish() + liveFinishContinuation = nil + samples = recorder.stop() + } let liveTask = liveCaptureTask liveCaptureTask = nil @@ -285,8 +318,14 @@ final class MacDictationViewModel: ObservableObject { guard let self else { return } do { let result: MacDictationResult + let capturedSamples: [Float] if let liveTask { let capture = await Self.awaitLiveCapture(liveTask) + if usesDeferredStop { + capturedSamples = self.recorder.stop() + } else { + capturedSamples = samples + } let trimmedLive = capture.raw.trimmingCharacters(in: .whitespacesAndNewlines) if !capture.shouldFallbackToBatch, !trimmedLive.isEmpty { if self.transcript.isEmpty { @@ -300,7 +339,7 @@ final class MacDictationViewModel: ObservableObject { ) } else { result = try await MacDictationPipeline.run( - samples: samples, + samples: capturedSamples, store: store, onPartial: { [weak self] partial in Task { @MainActor in @@ -310,8 +349,9 @@ final class MacDictationViewModel: ObservableObject { ) } } else { + capturedSamples = usesDeferredStop ? self.recorder.stop() : samples result = try await MacDictationPipeline.run( - samples: samples, + samples: capturedSamples, store: store, onPartial: { [weak self] partial in Task { @MainActor in @@ -324,6 +364,7 @@ final class MacDictationViewModel: ObservableObject { let pasted = try await self.deliver(result.text) self.recordUsage(for: result.text) self.speechHistory.append(text: result.text) + self.appendToOverview(result.text) self.statusMessage = self.statusAfterDelivery( pasted: pasted, polishWarning: result.polishWarning, @@ -332,17 +373,28 @@ final class MacDictationViewModel: ObservableObject { } catch { self.statusMessage = error.localizedDescription } + // The finalized take now lives in the overview; clear the live take + // so the HUD flashes its completion state and the next press starts + // fresh without overwriting the Home overview. + self.transcript = "" self.isStreamingPartial = false self.isProcessing = false + self.liveFinishContinuation?.finish() + self.liveFinishContinuation = nil } } private func startLiveCaptureIfSupported(store: AppGroupStore) { guard MacDictationPipeline.supportsLivePartials(store: store) else { return } let stream = recorder.makeSnapshotStream() + let (finishStream, finishContinuation) = AsyncStream.makeStream( + bufferingPolicy: .bufferingNewest(1) + ) + liveFinishContinuation = finishContinuation liveCaptureTask = Task { [weak self] in await MacDictationPipeline.captureLive( stream: stream, + finishSignal: finishStream, store: store, onPartial: { [weak self] partial in Task { @MainActor in @@ -359,6 +411,8 @@ final class MacDictationViewModel: ObservableObject { } private func cancelLiveCapture() { + liveFinishContinuation?.finish() + liveFinishContinuation = nil liveCaptureTask?.cancel() liveCaptureTask = nil isStreamingPartial = false @@ -475,6 +529,14 @@ final class MacDictationViewModel: ObservableObject { sessionTimer = nil } + private func appendToOverview(_ text: String) { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + overviewTranscript = overviewTranscript.isEmpty + ? trimmed + : overviewTranscript + "\n" + trimmed + } + private func recordUsage(for text: String) { usageStatistics.recordUtterance( text: text, diff --git a/OSGKeyboardMac/MacHallucinationFilter.swift b/OSGKeyboardMac/MacHallucinationFilter.swift new file mode 100644 index 0000000..5f88a2a --- /dev/null +++ b/OSGKeyboardMac/MacHallucinationFilter.swift @@ -0,0 +1,85 @@ +// MacHallucinationFilter.swift +// OSGKeyboard · Mac +// +// Strips Qwen3 / MLX streaming scaffold tokens and silence hallucinations. + +import Foundation + +enum MacQwen3LanguageHint { + /// Map persisted BCP-47 locale ids to Qwen3 prompt language names. + /// Returns `nil` for auto-detect. + static func from(locale: Locale) -> String? { + let raw = locale.identifier.lowercased() + if raw.isEmpty || raw == "auto" { return nil } + if raw.hasPrefix("zh") { return "Chinese" } + if raw.hasPrefix("en") { return "English" } + if raw.hasPrefix("ja") { return "Japanese" } + if raw.hasPrefix("ko") { return "Korean" } + if raw.hasPrefix("fr") { return "French" } + if raw.hasPrefix("de") { return "German" } + if raw.hasPrefix("es") { return "Spanish" } + if raw.hasPrefix("pt") { return "Portuguese" } + if raw.hasPrefix("ru") { return "Russian" } + if raw.hasPrefix("ar") { return "Arabic" } + return nil + } +} + +enum MacHallucinationFilter { + /// RMS below this skips feeding audio into the MLX streaming session. + static let silencePeakThreshold: Float = 0.0005 + + static func strip(_ raw: String) -> String { + var text = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if text.isEmpty { return "" } + + if let marker = text.range(of: "", options: .backwards) { + text = String(text[marker.upperBound...]) + .trimmingCharacters(in: .whitespacesAndNewlines) + } else if let match = text.range( + of: #"^language\s+\S+\s*"#, + options: [.regularExpression, .caseInsensitive] + ) { + text = String(text[match.upperBound...]) + .trimmingCharacters(in: .whitespacesAndNewlines) + } + + if isMetadataNoiseLine(text) { return "" } + return text + } + + /// When the transcript is mostly vocabulary tokens and audio energy stayed low, drop it. + static func shouldDiscardHotwordDump( + text: String, + peakRMS: Float, + bias: LocalASRBiasPayload? + ) -> Bool { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return true } + guard peakRMS < FlowCaptureTailDrainPolicy.flowDefault.silenceRMSThreshold else { + return false + } + guard let bias, !bias.hardHotwords.isEmpty else { return false } + let lowered = trimmed.lowercased() + let hits = bias.hardHotwords.filter { lowered.contains($0.lowercased()) }.count + let wordCount = max(1, trimmed.split { $0.isWhitespace }.count) + return hits >= wordCount + } + + private static func isMetadataNoiseLine(_ line: String) -> Bool { + let lowered = line.lowercased() + switch lowered { + case "language", "emotion", "event", "text", + "", "", "<|im_end|>": + return true + default: + if lowered.range( + of: #"^language(\s+\S+)?$"#, + options: .regularExpression + ) != nil { + return true + } + return false + } + } +} diff --git a/OSGKeyboardMac/MacLocalASRChunkAdapter.swift b/OSGKeyboardMac/MacLocalASRChunkAdapter.swift deleted file mode 100644 index a3b3f04..0000000 --- a/OSGKeyboardMac/MacLocalASRChunkAdapter.swift +++ /dev/null @@ -1,43 +0,0 @@ -// MacLocalASRChunkAdapter.swift -// OSGKeyboard · Mac -// -// Adapts macOS local ASR to the shared chunked utterance pipeline. - -import Foundation -import os - -final class MacLocalASRChunkAdapter: ASRChunkTranscribing, @unchecked Sendable { - private let locale: Locale - private let bias: LocalASRBiasPayload? - private let cancelled = OSAllocatedUnfairLock(initialState: false) - - init(locale: Locale, bias: LocalASRBiasPayload?) { - self.locale = locale - self.bias = bias - } - - func resetForNewUtterance() { - cancelled.withLock { $0 = false } - } - - func cancel() { - cancelled.withLock { $0 = true } - } - - func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult { - let isCancelled = cancelled.withLock { $0 } - if isCancelled || Task.isCancelled { return .cancelled } - guard !samples.isEmpty else { return .success("") } - - do { - let text = try await MacLocalASRService.transcribe( - samples: samples, - locale: locale, - bias: bias - ) - return .success(text) - } catch { - return .failure(error.localizedDescription) - } - } -} diff --git a/OSGKeyboardMac/MacLocalASRModelSettingsView.swift b/OSGKeyboardMac/MacLocalASRModelSettingsView.swift index 72f5b17..5921583 100644 --- a/OSGKeyboardMac/MacLocalASRModelSettingsView.swift +++ b/OSGKeyboardMac/MacLocalASRModelSettingsView.swift @@ -10,6 +10,7 @@ import SwiftUI final class MacLocalASRModelSettingsViewModel: ObservableObject { @Published var catalog: LocalASRCatalogDocument? @Published var selectedModelId: String = MacLocalASRPreferences.selectedModelId + @Published var downloadSource: LocalASRDownloadSourcePreference = MacLocalASRPreferences.downloadSource @Published var installProgress = LocalASRModelInstallProgress.idle @Published var diagnosticsSnapshot = LocalASRBiasDiagnosticsStore.load() @Published var statusMessage = "" @@ -76,15 +77,21 @@ final class MacLocalASRModelSettingsViewModel: ObservableObject { onLocalModelStateChanged?() } + func setDownloadSource(_ source: LocalASRDownloadSourcePreference) { + downloadSource = source + MacLocalASRPreferences.downloadSource = source + } + func installModel(_ model: LocalASRModelDefinition) { guard let catalog, !isInstalling else { return } statusMessage = "" isInstalling = true isDownloadPaused = false startProgressPolling() + let preferredSource = downloadSource Task { do { - try await manager.installModel(model, catalog: catalog) + try await manager.installModel(model, catalog: catalog, preferredSource: preferredSource) installProgress = await manager.currentProgress() selectModel(model.id) statusMessage = MacL10n.string("mac.localASR.installDone") @@ -222,18 +229,6 @@ struct MacLocalASRModelSettingsView: View { private func modelPickerSection(catalog: LocalASRCatalogDocument) -> some View { MacSettingsSection(title: MacL10n.string("mac.localASR.models", language: lang)) { VStack(spacing: MacMetrics.settingsRowGap) { - if let runtime = modelVM.currentRuntime(in: catalog) { - MacFormSubtitleRow(title: runtime.displayName) { - Text( - modelVM.isRuntimeInstalled(runtime) - ? MacL10n.string("mac.localASR.installed", language: lang) - : MacL10n.string("mac.localASR.notInstalled", language: lang) - ) - .font(TypeStyle.body) - .foregroundStyle(palette.textSecondary) - } - } - ForEach(Array(catalog.models.enumerated()), id: \.element.id) { _, model in modelRow(model) .frame(minHeight: MacMetrics.settingsRowMinHeight) @@ -260,6 +255,10 @@ struct MacLocalASRModelSettingsView: View { .padding(.horizontal, MacMetrics.settingsCardInset) } + downloadSourceRow + .frame(minHeight: MacMetrics.settingsRowMinHeight) + .padding(.horizontal, MacMetrics.settingsCardInset) + HStack(spacing: 0) { MacSettingsToolButton(title: MacL10n.string("mac.localASR.openStorage", language: lang)) { modelVM.revealStorageRoot() @@ -271,6 +270,29 @@ struct MacLocalASRModelSettingsView: View { } } + private var downloadSourceRow: some View { + HStack(spacing: Spacing.sm) { + Text(MacL10n.string("mac.localASR.downloadSource", language: lang)) + .foregroundStyle(palette.textSecondary) + Spacer(minLength: 0) + Picker("", selection: Binding( + get: { modelVM.downloadSource }, + set: { modelVM.setDownloadSource($0) } + )) { + Text(MacL10n.string("mac.localASR.downloadSource.auto", language: lang)) + .tag(LocalASRDownloadSourcePreference.auto) + Text(MacL10n.string("mac.localASR.downloadSource.hfMirror", language: lang)) + .tag(LocalASRDownloadSourcePreference.hfMirror) + Text(MacL10n.string("mac.localASR.downloadSource.huggingface", language: lang)) + .tag(LocalASRDownloadSourcePreference.huggingface) + } + .labelsHidden() + .pickerStyle(.menu) + .fixedSize() + .disabled(modelVM.isInstalling) + } + } + @ViewBuilder private func modelRow(_ model: LocalASRModelDefinition) -> some View { let installed = modelVM.isInstalled(model) diff --git a/OSGKeyboardMac/MacLocalASRService.swift b/OSGKeyboardMac/MacLocalASRService.swift index 7b11dd2..eccc0c7 100644 --- a/OSGKeyboardMac/MacLocalASRService.swift +++ b/OSGKeyboardMac/MacLocalASRService.swift @@ -2,14 +2,12 @@ // OSGKeyboard · Mac // // On-device ASR for macOS. Routes through the bundled local ASR catalog: -// Sherpa Qwen3 (default), Paraformer, SenseVoice, Apple Speech fallback. +// Qwen3 MLX streaming (default), Apple Speech fallback. import Foundation enum MacLocalASRBackend: String, Sendable, CaseIterable { - case sherpaQwen3 - case sherpaParaformer - case sherpaSenseVoice + case mlxQwen3 case appleSpeech } @@ -42,36 +40,53 @@ enum MacLocalASRError: Error, LocalizedError { enum MacLocalASRPreferences { static let backendKey = "mac.localASR.backend" static let selectedModelIdKey = LocalASRPreferenceKeys.selectedModelId - /// Legacy MLX path key — retained for migration only. - static let qwen3ModelRelativePath = "models/qwen3-asr-1.7b-mlx" + static let downloadSourceKey = LocalASRPreferenceKeys.downloadSource + + /// Preferred model download mirror; `.auto` picks by region (hf-mirror-friendly). + static var downloadSource: LocalASRDownloadSourcePreference { + get { + guard let raw = UserDefaults.standard.string(forKey: downloadSourceKey), + let value = LocalASRDownloadSourcePreference(rawValue: raw) else { + return .auto + } + return value + } + set { UserDefaults.standard.set(newValue.rawValue, forKey: downloadSourceKey) } + } static var selectedModelId: String { get { if let raw = UserDefaults.standard.string(forKey: selectedModelIdKey), !raw.isEmpty { return migratedModelId(raw) } - return legacyBackend == .appleSpeech ? "apple-speech-fallback" : "sherpa-qwen3-0.6b-int8" + return legacyBackend == .appleSpeech ? "apple-speech-fallback" : "qwen3-mlx-0.6b-4bit" } set { UserDefaults.standard.set(newValue, forKey: selectedModelIdKey) } } - /// Maps removed catalog entries to the current default Sherpa model. + /// Maps removed Sherpa / legacy catalog entries to the current MLX default. static func migratedModelId(_ id: String) -> String { switch id { - case "qwen3-mlx-1.7b", "sherpa-paraformer-zh-int8": - return "sherpa-qwen3-0.6b-int8" + case "sherpa-qwen3-0.6b-int8", + "sherpa-qwen3-1.7b-int8", + "sherpa-sensevoice-small-int8", + "sherpa-paraformer-zh-int8", + "qwen3-mlx-1.7b": + return "qwen3-mlx-0.6b-4bit" default: return id } } static var legacyBackend: MacLocalASRBackend { - guard let raw = UserDefaults.standard.string(forKey: backendKey), - let value = MacLocalASRBackend(rawValue: raw) else { - return .sherpaQwen3 + guard let raw = UserDefaults.standard.string(forKey: backendKey) else { + return .mlxQwen3 } - if raw == "qwen3MLX" { return .sherpaQwen3 } - return value + if raw == "qwen3MLX" || raw == "sherpaQwen3" || raw == "mlxQwen3" { + return .mlxQwen3 + } + if raw == "appleSpeech" { return .appleSpeech } + return .mlxQwen3 } } @@ -110,6 +125,12 @@ enum MacLocalASRService { ) } + /// Whether the active local engine uses MLX streaming for live partials. + static func usesMLXLiveStreaming() -> Bool { + guard let model = selectedModelDefinition() else { return false } + return model.backend == .mlx && isModelInstalled(model) + } + /// Transcribe using the selected catalog model, falling back to Apple Speech. static func transcribe( samples: [Float], @@ -131,17 +152,16 @@ enum MacLocalASRService { ) async throws -> String { switch model.backend { case .mlx: - throw MacLocalASRError.qwen3ModelMissing - case .sherpaQwen3, .sherpaSenseVoice, .sherpaParaformer: - return try await MacSherpaLocalASR.transcribe( + return try await MacMLXStreamingASRProvider.shared.transcribeBatch( samples: samples, - sampleRate: 16_000, - locale: locale, model: model, + locale: locale, bias: bias ) case .appleSpeech: return try await MacSpeechLocalASR.transcribe(samples: samples, locale: locale, bias: bias) + case .sherpaQwen3, .sherpaSenseVoice, .sherpaParaformer: + throw MacLocalASRError.qwen3ModelMissing } } } diff --git a/OSGKeyboardMac/MacMLXLiveCapture.swift b/OSGKeyboardMac/MacMLXLiveCapture.swift new file mode 100644 index 0000000..114853c --- /dev/null +++ b/OSGKeyboardMac/MacMLXLiveCapture.swift @@ -0,0 +1,150 @@ +// MacMLXLiveCapture.swift +// OSGKeyboard · Mac +// +// MLX streaming live capture: feed mic snapshots, tail drain, finalize. + +import Foundation +import os + +enum MacMLXLiveCapture { + private static let tailDrainPolicy = FlowCaptureTailDrainPolicy( + silenceRMSThreshold: 0.015, + silenceDurationSeconds: 0.35, + maxDrainSeconds: 0.75 + ) + + /// Runs MLX streaming ASR until `finishSignal` fires, then tail-drains and finalizes. + static func run( + audioStream: AsyncStream, + finishSignal: AsyncStream, + store: AppGroupStore, + onPartial: @escaping @Sendable (String) -> Void + ) async -> MacLiveASRCaptureResult { + let locale = Locale(identifier: store.localeId.isEmpty ? "zh-CN" : store.localeId) + let bias = resolveBias(store: store, locale: locale) + + guard let model = MacLocalASRService.selectedModelDefinition(), + model.backend == .mlx, + MacLocalASRService.isModelInstalled(model) else { + return MacLiveASRCaptureResult( + raw: "", + chunkWarning: nil, + localBias: bias, + shouldFallbackToBatch: true + ) + } + + do { + try await MacMLXStreamingASRProvider.shared.prepare(model: model) + let session = try await MacMLXStreamingASRProvider.shared.makeSession( + model: model, + bias: bias, + locale: locale + ) + session.onDisplayUpdate = { text in + onPartial(text) + } + + let drainTracker = FlowCaptureDrainTracker() + let draining = OSAllocatedUnfairLock(initialState: false) + let pendingFeed = OSAllocatedUnfairLock(initialState: [Float]()) + let feedIntervalSamples = 1_600 // 100 ms @ 16 kHz + + await withTaskGroup(of: Void.self) { group in + group.addTask { + // Only the first finish signal matters. The stream is only + // *yielded* to (never `finish()`ed) on the deferred-stop + // path, so without this `break` the loop would await a + // second element forever and hang the whole task group + // until the 120s hard timeout. + for await _ in finishSignal { + draining.withLock { $0 = true } + drainTracker.beginDrain() + break + } + } + + group.addTask { + for await snapshot in audioStream { + if Task.isCancelled { break } + if draining.withLock({ $0 }) { + drainTracker.noteAudio(samples: snapshot.samples, policy: tailDrainPolicy) + let decision = drainTracker.shouldFinish(policy: tailDrainPolicy) + if decision.finished { break } + } + pendingFeed.withLock { buffer in + buffer.append(contentsOf: snapshot.samples) + while buffer.count >= feedIntervalSamples { + let chunk = Array(buffer.prefix(feedIntervalSamples)) + buffer.removeFirst(feedIntervalSamples) + session.feed(samples: chunk) + } + } + } + } + } + + let remainder = pendingFeed.withLock { $0 } + if !remainder.isEmpty, !Task.isCancelled { + session.feed(samples: remainder) + } + + if Task.isCancelled { + session.cancel() + return MacLiveASRCaptureResult( + raw: "", + chunkWarning: nil, + localBias: bias, + shouldFallbackToBatch: true + ) + } + + let raw = try await session.stop() + if MacHallucinationFilter.shouldDiscardHotwordDump( + text: raw, + peakRMS: session.peakAudioRMS(), + bias: bias + ) { + return MacLiveASRCaptureResult( + raw: "", + chunkWarning: nil, + localBias: bias, + shouldFallbackToBatch: true + ) + } + + return MacLiveASRCaptureResult( + raw: raw, + chunkWarning: nil, + localBias: bias, + shouldFallbackToBatch: false + ) + } catch { + return MacLiveASRCaptureResult( + raw: "", + chunkWarning: nil, + localBias: bias, + shouldFallbackToBatch: true + ) + } + } + + private static func resolveBias(store: AppGroupStore, locale: Locale) -> LocalASRBiasPayload? { + MacAppContextService.captureAndPersist(to: store) + let capabilities = MacLocalASRService.currentCapabilities() + let bias = LocalASRBiasAdapter.adapt( + LocalASRBiasRequest( + dictionary: store.personalDictionary, + locale: locale, + frontAppBundleId: MacAppContextService.frontmostBundleIdentifier(), + capabilities: capabilities + ) + ) + LocalASRBiasDiagnosticsStore.save( + payload: bias, + modelId: MacLocalASRService.selectedModelDefinition()?.id, + backendLabel: MacLocalASRService.currentBackendLabel() + ) + return bias + } +} diff --git a/OSGKeyboardMac/MacMLXStreamingASRProvider.swift b/OSGKeyboardMac/MacMLXStreamingASRProvider.swift new file mode 100644 index 0000000..ff47813 --- /dev/null +++ b/OSGKeyboardMac/MacMLXStreamingASRProvider.swift @@ -0,0 +1,91 @@ +// MacMLXStreamingASRProvider.swift +// OSGKeyboard · Mac +// +// Loads and caches Qwen3 MLX models; builds streaming sessions with bias. + +import Foundation +import MLX +import MLXAudioSTT + +actor MacMLXStreamingASRProvider { + static let shared = MacMLXStreamingASRProvider() + + private var cachedModelId: String? + private var cachedModel: Qwen3ASRModel? + private var didWarmup = false + + func prepare(model: LocalASRModelDefinition) async throws { + _ = try await loadModel(model) + try await warmupIfNeeded() + } + + func makeSession( + model: LocalASRModelDefinition, + bias: LocalASRBiasPayload?, + locale: Locale + ) async throws -> MacMLXStreamingSession { + let qwen = try await loadModel(model) + var config = StreamingConfig( + decodeIntervalSeconds: 0.5, + boundaryDecodeIntervalSeconds: 0.2, + boundaryBoostSeconds: 1.0, + encoderWindowOverlapSeconds: 1.0, + maxCachedWindows: 8, + delayPreset: .realtime, + language: MacQwen3LanguageHint.from(locale: locale), + context: bias?.promptBias, + temperature: 0, + maxTokensPerPass: 512, + minAgreementPasses: 2, + boundaryMinAgreementPasses: 2, + maxDecodeWindows: 1, + finalizeCompletedWindows: true + ) + return MacMLXStreamingSession(model: qwen, config: config) + } + + func transcribeBatch( + samples: [Float], + model: LocalASRModelDefinition, + locale: Locale, + bias: LocalASRBiasPayload? + ) async throws -> String { + let qwen = try await loadModel(model) + let audio = MLXArray(samples.map { Float32($0) }) + let language = MacQwen3LanguageHint.from(locale: locale) + let context = bias?.promptBias ?? "" + let output = qwen.generate( + audio: audio, + context: context, + language: language + ) + let cleaned = MacHallucinationFilter.strip(output.text) + guard !cleaned.isEmpty else { throw MacLocalASRError.emptyTranscript } + return cleaned + } + + // MARK: - Private + + private func loadModel(_ definition: LocalASRModelDefinition) async throws -> Qwen3ASRModel { + if cachedModelId == definition.id, let cachedModel { + return cachedModel + } + guard let root = LocalASRModelInstallState.modelRootURL(definition) else { + throw MacLocalASRError.qwen3ModelMissing + } + let model = try await Qwen3ASRModel.fromModelDirectory(root) + cachedModelId = definition.id + cachedModel = model + didWarmup = false + return model + } + + private func warmupIfNeeded() async throws { + guard !didWarmup else { return } + guard let model = cachedModel else { return } + didWarmup = true + // One second of silence — primes Metal kernels before the first user take. + let silence = MLXArray([Float](repeating: 0, count: 16_000).map { Float32($0) }) + _ = model.generate(audio: silence, language: MacQwen3LanguageHint.from(locale: Locale(identifier: "zh-CN"))) + } +} diff --git a/OSGKeyboardMac/MacMLXStreamingSession.swift b/OSGKeyboardMac/MacMLXStreamingSession.swift new file mode 100644 index 0000000..61e4ce0 --- /dev/null +++ b/OSGKeyboardMac/MacMLXStreamingSession.swift @@ -0,0 +1,86 @@ +// MacMLXStreamingSession.swift +// OSGKeyboard · Mac +// +// Thin wrapper around mlx-audio-swift `StreamingInferenceSession`. + +import Foundation +import MLXAudioSTT + +/// One live MLX streaming ASR take (Option held). +final class MacMLXStreamingSession: @unchecked Sendable { + private let session: StreamingInferenceSession + private let eventTask: Task + private let lock = NSLock() + private var endedContinuation: CheckedContinuation? + private var peakRMS: Float = 0 + + var onDisplayUpdate: (@Sendable (String) -> Void)? + + init(model: Qwen3ASRModel, config: StreamingConfig) { + let session = StreamingInferenceSession(model: model, config: config) + self.session = session + + final class EventSink: @unchecked Sendable { + weak var owner: MacMLXStreamingSession? + } + let sink = EventSink() + self.eventTask = Task { + for await event in session.events { + sink.owner?.handle(event) + } + } + sink.owner = self + } + + func feed(samples: [Float]) { + guard !samples.isEmpty else { return } + let rms = FlowCaptureDrainTracker.rms(of: samples) + lock.withLock { + peakRMS = max(peakRMS, rms) + } + if rms < MacHallucinationFilter.silencePeakThreshold { return } + session.feedAudio(samples: samples) + } + + func stop() async throws -> String { + try await withCheckedThrowingContinuation { continuation in + lock.withLock { + endedContinuation = continuation + } + session.stop() + } + } + + func cancel() { + session.cancel() + lock.withLock { + endedContinuation?.resume(throwing: MacLocalASRError.qwen3InferenceFailed("Cancelled")) + endedContinuation = nil + } + eventTask.cancel() + } + + func peakAudioRMS() -> Float { + lock.withLock { peakRMS } + } + + private func handle(_ event: TranscriptionEvent) { + switch event { + case .displayUpdate(let confirmed, let provisional): + let display = confirmed + provisional + let cleaned = MacHallucinationFilter.strip(display) + guard !cleaned.isEmpty else { return } + onDisplayUpdate?(cleaned) + case .ended(let fullText): + let cleaned = MacHallucinationFilter.strip(fullText) + lock.withLock { + if let continuation = endedContinuation { + continuation.resume(returning: cleaned) + endedContinuation = nil + } + } + case .provisional, .confirmed, .stats: + break + } + } +} diff --git a/OSGKeyboardMac/MacSherpaLocalASR.swift b/OSGKeyboardMac/MacSherpaLocalASR.swift deleted file mode 100644 index 6bb7762..0000000 --- a/OSGKeyboardMac/MacSherpaLocalASR.swift +++ /dev/null @@ -1,64 +0,0 @@ -// MacSherpaLocalASR.swift -// OSGKeyboard · Mac -// -// Sherpa-onnx backed local ASR (Qwen3 hotwords POC + SenseVoice baseline). - -import Foundation - -enum MacSherpaLocalASR { - - static func transcribe( - samples: [Float], - sampleRate: Int, - locale: Locale, - model: LocalASRModelDefinition, - bias: LocalASRBiasPayload? - ) async throws -> String { - let catalog = try LocalASRModelCatalog.loadBundled() - let manager = LocalASRModelManager.shared - guard let layout = model.layout, - let modelRoot = LocalASRModelInstallState.modelRootURL(model) else { - throw MacLocalASRError.qwen3ModelMissing - } - - try await manager.ensureRuntimeInstalled(catalog: catalog) - guard let runtime = LocalASRModelCatalog.runtime( - for: LocalASRModelCatalog.currentRuntimePlatform(), - in: catalog - ), - let binary = LocalASRModelInstallState.resolveRuntimeBinary(runtime: runtime) else { - throw MacLocalASRError.qwen3LoadFailed("Sherpa runtime binary missing") - } - - switch model.backend { - case .sherpaQwen3: - return try await MacSherpaONNXRunner.transcribeQwen3( - samples: samples, - sampleRate: sampleRate, - locale: locale, - modelRoot: modelRoot, - layout: layout, - runtimeBinary: binary, - bias: bias - ) - case .sherpaSenseVoice: - return try await MacSherpaONNXRunner.transcribeSenseVoice( - samples: samples, - sampleRate: sampleRate, - modelRoot: modelRoot, - layout: layout, - runtimeBinary: binary - ) - case .sherpaParaformer: - return try await MacSherpaONNXRunner.transcribeParaformer( - samples: samples, - sampleRate: sampleRate, - modelRoot: modelRoot, - layout: layout, - runtimeBinary: binary - ) - default: - throw MacLocalASRError.qwen3InferenceFailed("Unsupported Sherpa backend") - } - } -} diff --git a/OSGKeyboardMac/MacSherpaONNXRunner.swift b/OSGKeyboardMac/MacSherpaONNXRunner.swift deleted file mode 100644 index 5602691..0000000 --- a/OSGKeyboardMac/MacSherpaONNXRunner.swift +++ /dev/null @@ -1,253 +0,0 @@ -// MacSherpaONNXRunner.swift -// OSGKeyboard · Mac -// -// Invokes the downloaded `sherpa-onnx-offline` binary for Sherpa-backed POC models. - -import Foundation - -enum MacSherpaONNXRunner { - - static func transcribeQwen3( - samples: [Float], - sampleRate: Int, - locale: Locale, - modelRoot: URL, - layout: LocalASRModelLayout, - runtimeBinary: URL, - bias: LocalASRBiasPayload? - ) async throws -> String { - guard sampleRate == 16_000 else { - throw MacLocalASRError.qwen3InferenceFailed("Sherpa expects 16 kHz audio") - } - guard let conv = layout.convFrontend, - let encoder = layout.encoder, - let decoder = layout.decoder, - let tokenizer = layout.tokenizer else { - throw MacLocalASRError.qwen3InferenceFailed("Incomplete Sherpa Qwen3 layout") - } - - let wavURL = try writeTemporaryWAV(samples: samples, sampleRate: sampleRate) - defer { try? FileManager.default.removeItem(at: wavURL) } - - var arguments = [ - "--qwen3-asr-conv-frontend=\(modelRoot.appendingPathComponent(conv).path)", - "--qwen3-asr-encoder=\(modelRoot.appendingPathComponent(encoder).path)", - "--qwen3-asr-decoder=\(modelRoot.appendingPathComponent(decoder).path)", - "--qwen3-asr-tokenizer=\(modelRoot.appendingPathComponent(tokenizer).path)", - "--qwen3-asr-max-new-tokens=512", - "--num-threads=2", - ] - - if let language = MacQwen3LanguageHint.from(locale: locale) { - arguments.append("--qwen3-asr-language=\(language)") - } - - if let hotwords = bias?.hardHotwords, !hotwords.isEmpty { - arguments.append("--qwen3-asr-hotwords=\(hotwords.joined(separator: ","))") - } - - arguments.append(wavURL.path) - return try await run(binary: runtimeBinary, arguments: arguments) - } - - static func transcribeSenseVoice( - samples: [Float], - sampleRate: Int, - modelRoot: URL, - layout: LocalASRModelLayout, - runtimeBinary: URL - ) async throws -> String { - guard sampleRate == 16_000 else { - throw MacLocalASRError.qwen3InferenceFailed("Sherpa expects 16 kHz audio") - } - guard let model = layout.senseVoiceModel, - let tokens = layout.tokens else { - throw MacLocalASRError.qwen3InferenceFailed("Incomplete SenseVoice layout") - } - - let wavURL = try writeTemporaryWAV(samples: samples, sampleRate: sampleRate) - defer { try? FileManager.default.removeItem(at: wavURL) } - - let arguments = [ - "--tokens=\(modelRoot.appendingPathComponent(tokens).path)", - "--sense-voice-model=\(modelRoot.appendingPathComponent(model).path)", - "--num-threads=2", - wavURL.path, - ] - return try await run(binary: runtimeBinary, arguments: arguments) - } - - static func transcribeParaformer( - samples: [Float], - sampleRate: Int, - modelRoot: URL, - layout: LocalASRModelLayout, - runtimeBinary: URL - ) async throws -> String { - guard sampleRate == 16_000 else { - throw MacLocalASRError.qwen3InferenceFailed("Sherpa expects 16 kHz audio") - } - guard let paraformer = layout.paraformerModel, - let tokens = layout.tokens else { - throw MacLocalASRError.qwen3InferenceFailed("Incomplete Paraformer layout") - } - - let wavURL = try writeTemporaryWAV(samples: samples, sampleRate: sampleRate) - defer { try? FileManager.default.removeItem(at: wavURL) } - - let arguments = [ - "--tokens=\(modelRoot.appendingPathComponent(tokens).path)", - "--paraformer=\(modelRoot.appendingPathComponent(paraformer).path)", - "--num-threads=2", - wavURL.path, - ] - return try await run(binary: runtimeBinary, arguments: arguments) - } - - // MARK: - Private - - private static func writeTemporaryWAV(samples: [Float], sampleRate: Int) throws -> URL { - let data = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate) - let url = FileManager.default.temporaryDirectory - .appendingPathComponent("osg-sherpa-\(UUID().uuidString).wav") - try data.write(to: url, options: .atomic) - return url - } - - private static func run(binary: URL, arguments: [String]) async throws -> String { - try await withCheckedThrowingContinuation { continuation in - let process = Process() - process.executableURL = binary - process.arguments = arguments - process.currentDirectoryURL = binary.deletingLastPathComponent() - - let outputPipe = Pipe() - let errorPipe = Pipe() - process.standardOutput = outputPipe - process.standardError = errorPipe - - process.terminationHandler = { proc in - let outputData = outputPipe.fileHandleForReading.readDataToEndOfFile() - let errorData = errorPipe.fileHandleForReading.readDataToEndOfFile() - let stdout = String(data: outputData, encoding: .utf8) ?? "" - let stderr = String(data: errorData, encoding: .utf8) ?? "" - - guard proc.terminationStatus == 0 else { - let detail = stderr.isEmpty ? stdout : stderr - continuation.resume( - throwing: MacLocalASRError.qwen3InferenceFailed( - detail.trimmingCharacters(in: .whitespacesAndNewlines) - ) - ) - return - } - - let text = parseTranscript(stdout: stdout) - if text.isEmpty { - continuation.resume(throwing: MacLocalASRError.emptyTranscript) - } else { - continuation.resume(returning: text) - } - } - - do { - try process.run() - } catch { - continuation.resume(throwing: MacLocalASRError.qwen3InferenceFailed(error.localizedDescription)) - } - } - } - - private static func parseTranscript(stdout: String) -> String { - let lines = stdout - .split(whereSeparator: \.isNewline) - .map { String($0).trimmingCharacters(in: .whitespacesAndNewlines) } - .filter { !$0.isEmpty } - - for line in lines.reversed() { - if line.hasPrefix("{") { - // Sherpa's JSON result line (`{"text": ..., "lang": ..., ...}`). - // Trust only its `text` field — including when it's empty - // (silence/no-speech) — and never fall through to the raw - // JSON below, or the JSON blob itself gets inserted as text. - if let data = line.data(using: .utf8), - let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let text = object["text"] as? String { - return sanitizeTranscript(text) - } - continue - } - if isMetadataNoiseLine(line) { continue } - if !line.hasPrefix("/"), !line.hasPrefix("--"), line.count > 1 { - let cleaned = sanitizeTranscript(line) - if !cleaned.isEmpty { return cleaned } - } - } - return "" - } - - /// Qwen3-ASR (via sherpa-onnx) often prefixes the transcript with a - /// scaffold such as `language Chinese…`. Older runtimes leave - /// that intact in `result.text`; incomplete generations can even stop at - /// the bare word `language`. Strip the scaffold so only spoken text remains. - private static func sanitizeTranscript(_ raw: String) -> String { - var text = raw.trimmingCharacters(in: .whitespacesAndNewlines) - if text.isEmpty { return "" } - - // Prefer the payload after the last `` marker. - if let marker = text.range(of: "", options: .backwards) { - text = String(text[marker.upperBound...]) - .trimmingCharacters(in: .whitespacesAndNewlines) - } else if let match = text.range( - of: #"^language\s+\S+\s*"#, - options: [.regularExpression, .caseInsensitive] - ) { - // Fallback when the marker token was lost but the language prefix remains. - text = String(text[match.upperBound...]) - .trimmingCharacters(in: .whitespacesAndNewlines) - } - - // Drop leftover control tokens / bare scaffold words. - if isMetadataNoiseLine(text) { return "" } - return text - } - - /// Lines that are sherpa/Qwen metadata rather than spoken content. - private static func isMetadataNoiseLine(_ line: String) -> Bool { - let lowered = line.lowercased() - switch lowered { - case "language", "emotion", "event", "text", - "", "", "<|im_end|>": - return true - default: - // Exact scaffold with no spoken payload, e.g. "language Chinese". - if lowered.range( - of: #"^language(\s+\S+)?$"#, - options: .regularExpression - ) != nil { - return true - } - return false - } - } -} - -enum MacQwen3LanguageHint { - /// Map persisted BCP-47 locale ids to Qwen3 prompt language names. - /// Returns `nil` for auto-detect. - static func from(locale: Locale) -> String? { - let raw = locale.identifier.lowercased() - if raw.isEmpty || raw == "auto" { return nil } - if raw.hasPrefix("zh") { return "Chinese" } - if raw.hasPrefix("en") { return "English" } - if raw.hasPrefix("ja") { return "Japanese" } - if raw.hasPrefix("ko") { return "Korean" } - if raw.hasPrefix("fr") { return "French" } - if raw.hasPrefix("de") { return "German" } - if raw.hasPrefix("es") { return "Spanish" } - if raw.hasPrefix("pt") { return "Portuguese" } - if raw.hasPrefix("ru") { return "Russian" } - if raw.hasPrefix("ar") { return "Arabic" } - return nil - } -} diff --git a/OSGKeyboardShared/Models/LocalASRCapabilities.swift b/OSGKeyboardShared/Models/LocalASRCapabilities.swift index 37743c5..e5d9151 100644 --- a/OSGKeyboardShared/Models/LocalASRCapabilities.swift +++ b/OSGKeyboardShared/Models/LocalASRCapabilities.swift @@ -44,12 +44,12 @@ public struct LocalASRCapabilities: Sendable, Equatable { self.hotwordReloadCost = hotwordReloadCost } - /// Qwen3 MLX via mlx-swift-asr — `context` soft prompt on `transcribe`. + /// Qwen3 MLX via mlx-audio-swift — `context` soft prompt on streaming + batch. public static let qwen3MLX = LocalASRCapabilities( hotwordMode: .promptOnly, maxHotwordCount: 0, maxPromptCharacters: 800, - supportsStreaming: false, + supportsStreaming: true, hotwordReloadCost: .none ) diff --git a/OSGKeyboardShared/Models/LocalASRModelCatalog.swift b/OSGKeyboardShared/Models/LocalASRModelCatalog.swift index 614e032..607a9d1 100644 --- a/OSGKeyboardShared/Models/LocalASRModelCatalog.swift +++ b/OSGKeyboardShared/Models/LocalASRModelCatalog.swift @@ -53,6 +53,8 @@ public struct LocalASRModelLayout: Codable, Sendable, Equatable { public var tokenizer: String? public var senseVoiceModel: String? public var paraformerModel: String? + public var mlxConfig: String? + public var mlxWeights: String? public var tokens: String? } @@ -145,6 +147,25 @@ public enum LocalASRModelCatalog { #endif } +/// User-facing override for which mirror to try first when installing models. +public enum LocalASRDownloadSourcePreference: String, Codable, Sendable, CaseIterable { + /// Pick automatically based on the system region (China-friendly default). + case auto + /// Force the HF mirror (`hf-mirror.com`) first — best for mainland China. + case hfMirror + /// Force the official Hugging Face endpoint first. + case huggingface + + /// The catalog `type` string this preference pins to the front (nil for `.auto`). + public var pinnedType: String? { + switch self { + case .auto: return nil + case .hfMirror: return "hfmirror" + case .huggingface: return "huggingface" + } + } +} + /// Region-aware ordering for local ASR model download mirrors. public enum LocalASRDownloadSourceSorter { @@ -153,31 +174,49 @@ public enum LocalASRDownloadSourceSorter { region?.identifier == "CN" } - /// Lower rank = tried earlier. CN: ModelScope → HuggingFace → GitHub; elsewhere: HF → GitHub → ModelScope. + /// Whether to try the China-friendly mirror (`hf-mirror.com`) first. + /// + /// Default is "yes unless the region is *definitely* overseas": an unknown + /// region (common when a VPN/proxy masks locale) falls back to the mirror, + /// which is reachable both inside and outside China, so mainland users work + /// out of the box while overseas users only lose it when their region is set. + public static func preferChinaMirror(region: Locale.Region? = Locale.current.region) -> Bool { + guard let region else { return true } + return region.identifier == "CN" + } + + /// Lower rank = tried earlier. + /// China-first: hf-mirror → HuggingFace → ModelScope → GitHub. + /// Overseas: HuggingFace → hf-mirror → GitHub → ModelScope. public static func typeRank(_ type: String, chinaFirst: Bool) -> Int { switch type.lowercased() { - case "modelscope": - return chinaFirst ? 0 : 2 + case "hfmirror", "hf-mirror": + return chinaFirst ? 0 : 1 case "huggingface": return chinaFirst ? 1 : 0 + case "modelscope": + return chinaFirst ? 2 : 3 case "github": - return 1 + return chinaFirst ? 3 : 2 default: - return 3 + return 4 } } public static func sorted( _ sources: [LocalASRDownloadSource], - region: Locale.Region? = Locale.current.region + region: Locale.Region? = Locale.current.region, + preferred: LocalASRDownloadSourcePreference = .auto ) -> [LocalASRDownloadSource] { - let chinaFirst = isChinaMainland(region: region) - return sources.sorted { lhs, rhs in - let leftRank = typeRank(lhs.type, chinaFirst: chinaFirst) - let rightRank = typeRank(rhs.type, chinaFirst: chinaFirst) - if leftRank != rightRank { return leftRank < rightRank } - return lhs.priority < rhs.priority + let chinaFirst = preferChinaMirror(region: region) + let pinned = preferred.pinnedType?.lowercased() + func rank(_ source: LocalASRDownloadSource) -> (Int, Int) { + if let pinned, source.type.lowercased() == pinned { + return (-1, source.priority) + } + return (typeRank(source.type, chinaFirst: chinaFirst), source.priority) } + return sources.sorted { rank($0) < rank($1) } } } diff --git a/OSGKeyboardShared/Resources/LocalASR/local-asr-catalog.json b/OSGKeyboardShared/Resources/LocalASR/local-asr-catalog.json index 0a92e13..bd7bd76 100644 --- a/OSGKeyboardShared/Resources/LocalASR/local-asr-catalog.json +++ b/OSGKeyboardShared/Resources/LocalASR/local-asr-catalog.json @@ -1,142 +1,109 @@ { "schemaVersion": 1, - "defaultModelId": "sherpa-qwen3-0.6b-int8", - "runtimes": [ - { - "id": "sherpa-onnx-1.13.4-macos-arm64", - "displayName": "sherpa-onnx 1.13.4 (Apple Silicon)", - "installRelativePath": "runtimes/sherpa-onnx-1.13.4-macos-arm64", - "binaryCandidates": ["bin/sherpa-onnx-offline", "sherpa-onnx-offline"], - "archiveFileName": "sherpa-onnx-v1.13.4-osx-arm64-static-no-tts.tar.bz2", - "sizeBytes": 120000000, - "platform": "macos-arm64", - "sources": [ - { - "type": "github", - "priority": 1, - "url": "https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.13.4/sherpa-onnx-v1.13.4-osx-arm64-static-no-tts.tar.bz2" - } - ] - }, - { - "id": "sherpa-onnx-1.13.4-macos-x64", - "displayName": "sherpa-onnx 1.13.4 (Intel)", - "installRelativePath": "runtimes/sherpa-onnx-1.13.4-macos-x64", - "binaryCandidates": ["bin/sherpa-onnx-offline", "sherpa-onnx-offline"], - "archiveFileName": "sherpa-onnx-v1.13.4-osx-x64-static-no-tts.tar.bz2", - "sizeBytes": 130000000, - "platform": "macos-x64", - "sources": [ - { - "type": "github", - "priority": 1, - "url": "https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.13.4/sherpa-onnx-v1.13.4-osx-x64-static-no-tts.tar.bz2" - } - ] - } - ], + "defaultModelId": "qwen3-mlx-0.6b-4bit", + "runtimes": [], "models": [ { - "id": "sherpa-qwen3-0.6b-int8", + "id": "qwen3-mlx-0.6b-4bit", "displayName": "Qwen3-ASR 0.6B", - "backend": "sherpaQwen3", + "backend": "mlx", "runtimePlatform": "macos", - "sizeBytes": 650000000, + "sizeBytes": 730000000, "recommendedLocales": ["zh-CN", "en-US"], "supportsHotwords": true, - "hotwordMode": "recognizerScoped", + "hotwordMode": "promptOnly", "badgeKey": "mac.localASR.badge.balanced", - "installKind": "archive", - "installRelativePath": "models/sherpa-qwen3-0.6b-int8", - "archiveBaseName": "sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25", - "layout": { - "convFrontend": "conv_frontend.onnx", - "encoder": "encoder.int8.onnx", - "decoder": "decoder.int8.onnx", - "tokenizer": "tokenizer" - }, - "sources": [ - { - "type": "github", - "priority": 1, - "url": "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25.tar.bz2" - } - ] - }, - { - "id": "sherpa-qwen3-1.7b-int8", - "displayName": "Qwen3-ASR 1.7B", - "backend": "sherpaQwen3", - "runtimePlatform": "macos", - "sizeBytes": 1900000000, - "recommendedLocales": ["zh-CN", "en-US"], - "supportsHotwords": true, - "hotwordMode": "recognizerScoped", - "badgeKey": "mac.localASR.badge.quality", "installKind": "repository", - "installRelativePath": "models/sherpa-qwen3-1.7b-int8", - "archiveBaseName": "sherpa-onnx-qwen3-asr-1.7B-int8", + "installRelativePath": "models/qwen3-mlx-0.6b-4bit", + "archiveBaseName": "Qwen3-ASR-0.6B-4bit", "layout": { - "convFrontend": "conv_frontend.onnx", - "encoder": "encoder.int8.onnx", - "decoder": "decoder.int8.onnx", - "tokenizer": "tokenizer" + "mlxConfig": "config.json", + "mlxWeights": "model.safetensors" }, "sources": [ { - "type": "modelscope", + "type": "hfmirror", "priority": 1, "url": "", - "baseURL": "https://www.modelscope.cn/models/zengshuishui/Qwen3-ASR-onnx/resolve/master/{path}", + "baseURL": "https://hf-mirror.com/mlx-community/Qwen3-ASR-0.6B-4bit/resolve/main/{path}", "files": [ - { "remotePath": "model_1.7B/conv_frontend.onnx", "localPath": "conv_frontend.onnx", "sizeBytes": 12000000 }, - { "remotePath": "model_1.7B/encoder.int8.onnx", "localPath": "encoder.int8.onnx", "sizeBytes": 900000000 }, - { "remotePath": "model_1.7B/decoder.int8.onnx", "localPath": "decoder.int8.onnx", "sizeBytes": 700000000 }, - { "remotePath": "model_1.7B/tokenizer/merges.txt", "localPath": "tokenizer/merges.txt", "sizeBytes": 500000 }, - { "remotePath": "model_1.7B/tokenizer/vocab.json", "localPath": "tokenizer/vocab.json", "sizeBytes": 3000000 }, - { "remotePath": "model_1.7B/tokenizer/tokenizer.json", "localPath": "tokenizer/tokenizer.json", "sizeBytes": 7000000 }, - { "remotePath": "model_1.7B/tokenizer/tokenizer_config.json", "localPath": "tokenizer/tokenizer_config.json", "sizeBytes": 10000 } + { "remotePath": "config.json", "localPath": "config.json", "sizeBytes": 5000 }, + { "remotePath": "generation_config.json", "localPath": "generation_config.json", "sizeBytes": 200 }, + { "remotePath": "preprocessor_config.json", "localPath": "preprocessor_config.json", "sizeBytes": 500 }, + { "remotePath": "model.safetensors", "localPath": "model.safetensors", "sizeBytes": 708236945 }, + { "remotePath": "model.safetensors.index.json", "localPath": "model.safetensors.index.json", "sizeBytes": 80000 }, + { "remotePath": "tokenizer_config.json", "localPath": "tokenizer_config.json", "sizeBytes": 10000 }, + { "remotePath": "merges.txt", "localPath": "merges.txt", "sizeBytes": 1700000 }, + { "remotePath": "vocab.json", "localPath": "vocab.json", "sizeBytes": 2800000 } ] }, { "type": "huggingface", "priority": 1, "url": "", - "baseURL": "https://huggingface.co/zengshuishui/Qwen3-ASR-onnx/resolve/main/{path}", + "baseURL": "https://huggingface.co/mlx-community/Qwen3-ASR-0.6B-4bit/resolve/main/{path}", "files": [ - { "remotePath": "model_1.7B/conv_frontend.onnx", "localPath": "conv_frontend.onnx", "sizeBytes": 12000000 }, - { "remotePath": "model_1.7B/encoder.int8.onnx", "localPath": "encoder.int8.onnx", "sizeBytes": 900000000 }, - { "remotePath": "model_1.7B/decoder.int8.onnx", "localPath": "decoder.int8.onnx", "sizeBytes": 700000000 }, - { "remotePath": "model_1.7B/tokenizer/merges.txt", "localPath": "tokenizer/merges.txt", "sizeBytes": 500000 }, - { "remotePath": "model_1.7B/tokenizer/vocab.json", "localPath": "tokenizer/vocab.json", "sizeBytes": 3000000 }, - { "remotePath": "model_1.7B/tokenizer/tokenizer.json", "localPath": "tokenizer/tokenizer.json", "sizeBytes": 7000000 }, - { "remotePath": "model_1.7B/tokenizer/tokenizer_config.json", "localPath": "tokenizer/tokenizer_config.json", "sizeBytes": 10000 } + { "remotePath": "config.json", "localPath": "config.json", "sizeBytes": 5000 }, + { "remotePath": "generation_config.json", "localPath": "generation_config.json", "sizeBytes": 200 }, + { "remotePath": "preprocessor_config.json", "localPath": "preprocessor_config.json", "sizeBytes": 500 }, + { "remotePath": "model.safetensors", "localPath": "model.safetensors", "sizeBytes": 708236945 }, + { "remotePath": "model.safetensors.index.json", "localPath": "model.safetensors.index.json", "sizeBytes": 80000 }, + { "remotePath": "tokenizer_config.json", "localPath": "tokenizer_config.json", "sizeBytes": 10000 }, + { "remotePath": "merges.txt", "localPath": "merges.txt", "sizeBytes": 1700000 }, + { "remotePath": "vocab.json", "localPath": "vocab.json", "sizeBytes": 2800000 } ] } ] }, { - "id": "sherpa-sensevoice-small-int8", - "displayName": "SenseVoice Small", - "backend": "sherpaSenseVoice", + "id": "qwen3-mlx-1.7b-4bit", + "displayName": "Qwen3-ASR 1.7B", + "backend": "mlx", "runtimePlatform": "macos", - "sizeBytes": 250000000, - "recommendedLocales": ["zh-CN", "en-US", "ja-JP", "ko-KR"], - "supportsHotwords": false, - "hotwordMode": "none", - "badgeKey": "mac.localASR.badge.fastest", - "installKind": "archive", - "installRelativePath": "models/sherpa-sensevoice-small-int8", - "archiveBaseName": "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17", + "sizeBytes": 1700000000, + "recommendedLocales": ["zh-CN", "en-US"], + "supportsHotwords": true, + "hotwordMode": "promptOnly", + "badgeKey": "mac.localASR.badge.quality", + "installKind": "repository", + "installRelativePath": "models/qwen3-mlx-1.7b-4bit", + "archiveBaseName": "Qwen3-ASR-1.7B-4bit", "layout": { - "senseVoiceModel": "model.int8.onnx", - "tokens": "tokens.txt" + "mlxConfig": "config.json", + "mlxWeights": "model.safetensors" }, "sources": [ { - "type": "github", + "type": "hfmirror", "priority": 1, - "url": "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2" + "url": "", + "baseURL": "https://hf-mirror.com/mlx-community/Qwen3-ASR-1.7B-4bit/resolve/main/{path}", + "files": [ + { "remotePath": "config.json", "localPath": "config.json", "sizeBytes": 5000 }, + { "remotePath": "generation_config.json", "localPath": "generation_config.json", "sizeBytes": 200 }, + { "remotePath": "preprocessor_config.json", "localPath": "preprocessor_config.json", "sizeBytes": 500 }, + { "remotePath": "model.safetensors", "localPath": "model.safetensors", "sizeBytes": 1650000000 }, + { "remotePath": "model.safetensors.index.json", "localPath": "model.safetensors.index.json", "sizeBytes": 90000 }, + { "remotePath": "tokenizer_config.json", "localPath": "tokenizer_config.json", "sizeBytes": 10000 }, + { "remotePath": "merges.txt", "localPath": "merges.txt", "sizeBytes": 1700000 }, + { "remotePath": "vocab.json", "localPath": "vocab.json", "sizeBytes": 2800000 } + ] + }, + { + "type": "huggingface", + "priority": 1, + "url": "", + "baseURL": "https://huggingface.co/mlx-community/Qwen3-ASR-1.7B-4bit/resolve/main/{path}", + "files": [ + { "remotePath": "config.json", "localPath": "config.json", "sizeBytes": 5000 }, + { "remotePath": "generation_config.json", "localPath": "generation_config.json", "sizeBytes": 200 }, + { "remotePath": "preprocessor_config.json", "localPath": "preprocessor_config.json", "sizeBytes": 500 }, + { "remotePath": "model.safetensors", "localPath": "model.safetensors", "sizeBytes": 1650000000 }, + { "remotePath": "model.safetensors.index.json", "localPath": "model.safetensors.index.json", "sizeBytes": 90000 }, + { "remotePath": "tokenizer_config.json", "localPath": "tokenizer_config.json", "sizeBytes": 10000 }, + { "remotePath": "merges.txt", "localPath": "merges.txt", "sizeBytes": 1700000 }, + { "remotePath": "vocab.json", "localPath": "vocab.json", "sizeBytes": 2800000 } + ] } ] } diff --git a/OSGKeyboardShared/Services/LocalASRModelDownloadClient.swift b/OSGKeyboardShared/Services/LocalASRModelDownloadClient.swift index 1d5e809..a076db6 100644 --- a/OSGKeyboardShared/Services/LocalASRModelDownloadClient.swift +++ b/OSGKeyboardShared/Services/LocalASRModelDownloadClient.swift @@ -17,12 +17,20 @@ public struct LocalASRDownloadProgressUpdate: Sendable { } } -/// Controls an in-flight URLSession download; supports pause via resume data. +/// Controls an in-flight URLSession download; supports pause via resume data, +/// plus automatic retry-with-resume on transient network failures. public final class LocalASRModelDownloadController: NSObject, URLSessionDownloadDelegate, @unchecked Sendable { private let destinationURL: URL private let onProgress: @Sendable (LocalASRDownloadProgressUpdate) -> Void + private let maxRetries: Int private lazy var delegateSession: URLSession = { - URLSession(configuration: .default, delegate: self, delegateQueue: nil) + let config = URLSessionConfiguration.default + // Big weight files over flaky links: allow long total transfers but + // fail (and retry) a stalled connection that goes quiet for a while. + config.timeoutIntervalForRequest = 90 + config.timeoutIntervalForResource = 24 * 60 * 60 + config.waitsForConnectivity = true + return URLSession(configuration: config, delegate: self, delegateQueue: nil) }() private var remoteURL: URL? @@ -30,12 +38,15 @@ public final class LocalASRModelDownloadController: NSObject, URLSessionDownload private var completionContinuation: CheckedContinuation? private var isPausing = false private var finished = false + private var retryCount = 0 init( destinationURL: URL, + maxRetries: Int = 4, onProgress: @escaping @Sendable (LocalASRDownloadProgressUpdate) -> Void ) { self.destinationURL = destinationURL + self.maxRetries = maxRetries self.onProgress = onProgress super.init() } @@ -43,6 +54,7 @@ public final class LocalASRModelDownloadController: NSObject, URLSessionDownload /// Runs until the archive is fully written to `destinationURL` (survives pause/resume). public func download(from remoteURL: URL) async throws { self.remoteURL = remoteURL + retryCount = 0 try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in completionContinuation = continuation startTask(resumeData: nil) @@ -133,14 +145,55 @@ public final class LocalASRModelDownloadController: NSObject, URLSessionDownload public func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) { guard !finished else { return } if isPausing { return } - if let error { - finished = true - completionContinuation?.resume( - throwing: LocalASRModelManagerError.downloadFailed(error.localizedDescription) - ) - completionContinuation = nil - session.finishTasksAndInvalidate() + guard let error else { return } + + // Transient network drop: resume from where we stopped (if the server + // handed back resume data) after a short exponential backoff, up to a cap. + if Self.isRetryable(error), retryCount < maxRetries { + retryCount += 1 + let resumeData = (error as NSError) + .userInfo[NSURLSessionDownloadTaskResumeData] as? Data + let delay = Self.backoffSeconds(attempt: retryCount) + DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + delay) { [weak self] in + guard let self, !self.finished, !self.isPausing else { return } + self.startTask(resumeData: resumeData) + } + return } + + finished = true + completionContinuation?.resume( + throwing: LocalASRModelManagerError.downloadFailed(error.localizedDescription) + ) + completionContinuation = nil + session.finishTasksAndInvalidate() + } + + /// Network hiccups worth retrying; permanent failures (404, cancelled) are not. + private static func isRetryable(_ error: Error) -> Bool { + let nsError = error as NSError + guard nsError.domain == NSURLErrorDomain else { return false } + switch nsError.code { + case NSURLErrorNetworkConnectionLost, + NSURLErrorTimedOut, + NSURLErrorCannotConnectToHost, + NSURLErrorCannotFindHost, + NSURLErrorDNSLookupFailed, + NSURLErrorNotConnectedToInternet, + NSURLErrorSecureConnectionFailed, + NSURLErrorResourceUnavailable, + NSURLErrorHTTPTooManyRedirects, + NSURLErrorDataLengthExceedsMaximum, + NSURLErrorZeroByteResource: + return true + default: + return false + } + } + + /// 1s, 2s, 4s, 8s … capped at 30s. + private static func backoffSeconds(attempt: Int) -> Double { + min(30, pow(2, Double(attempt - 1))) } } @@ -148,9 +201,14 @@ public enum LocalASRModelDownloadClient { public static func makeController( destinationURL: URL, + maxRetries: Int = 4, onProgress: @escaping @Sendable (LocalASRDownloadProgressUpdate) -> Void ) -> LocalASRModelDownloadController { - LocalASRModelDownloadController(destinationURL: destinationURL, onProgress: onProgress) + LocalASRModelDownloadController( + destinationURL: destinationURL, + maxRetries: maxRetries, + onProgress: onProgress + ) } } diff --git a/OSGKeyboardShared/Services/LocalASRModelInstallState.swift b/OSGKeyboardShared/Services/LocalASRModelInstallState.swift index 6d6bd77..67f6a61 100644 --- a/OSGKeyboardShared/Services/LocalASRModelInstallState.swift +++ b/OSGKeyboardShared/Services/LocalASRModelInstallState.swift @@ -84,6 +84,11 @@ public enum LocalASRModelInstallState { fileManager: FileManager ) -> Bool { switch model.backend { + case .mlx: + guard let config = layout.mlxConfig, + let weights = layout.mlxWeights else { return false } + return fileManager.fileExists(atPath: root.appendingPathComponent(config).path) + && fileManager.fileExists(atPath: root.appendingPathComponent(weights).path) case .sherpaQwen3: guard let conv = layout.convFrontend, let encoder = layout.encoder, diff --git a/OSGKeyboardShared/Services/LocalASRModelManager.swift b/OSGKeyboardShared/Services/LocalASRModelManager.swift index a0580eb..6b31f3f 100644 --- a/OSGKeyboardShared/Services/LocalASRModelManager.swift +++ b/OSGKeyboardShared/Services/LocalASRModelManager.swift @@ -182,7 +182,8 @@ public actor LocalASRModelManager { public func installModel( _ model: LocalASRModelDefinition, - catalog: LocalASRCatalogDocument + catalog: LocalASRCatalogDocument, + preferredSource: LocalASRDownloadSourcePreference = .auto ) async throws { guard let relative = model.installRelativePath, let sources = model.sources, @@ -196,11 +197,7 @@ public actor LocalASRModelManager { message: model.displayName, activeItemId: model.id ) - if model.backend == .sherpaQwen3 || model.backend == .sherpaSenseVoice || model.backend == .sherpaParaformer { - try await ensureRuntimeInstalled(catalog: catalog) - } - - let sortedSources = LocalASRDownloadSourceSorter.sorted(sources) + let sortedSources = LocalASRDownloadSourceSorter.sorted(sources, preferred: preferredSource) var lastError: Error? switch model.installKind { @@ -518,9 +515,10 @@ public actor LocalASRModelManager { try fileManager.createDirectory(at: stagingRoot, withIntermediateDirectories: true) defer { try? fileManager.removeItem(at: stagingRoot) } - if fileManager.fileExists(atPath: destinationRoot.path) { - try fileManager.removeItem(at: destinationRoot) - } + // Do NOT wipe an existing destination: files land here only after a full + // download completes (partials stay in URLSession's temp dir), so already + // present files are complete and can be reused when a prior attempt failed + // partway or we fall back to another mirror. try fileManager.createDirectory(at: destinationRoot, withIntermediateDirectories: true) let totalBytes = files.reduce(Int64(0)) { partial, file in @@ -529,12 +527,21 @@ public actor LocalASRModelManager { var completedBytes: Int64 = 0 for (index, file) in files.enumerated() { + let localURL = destinationRoot.appendingPathComponent(file.localPath) + + // Skip files a previous attempt already finished (non-empty on disk). + if fileManager.fileExists(atPath: localURL.path), + let attrs = try? fileManager.attributesOfItem(atPath: localURL.path), + let size = attrs[.size] as? Int64, size > 0 { + completedBytes += Int64(file.sizeBytes ?? Int(size)) + continue + } + let remoteURLString = baseURL.replacingOccurrences(of: "{path}", with: file.remotePath) guard let remoteURL = URL(string: remoteURLString) else { throw LocalASRModelManagerError.downloadFailed("Invalid URL for \(file.remotePath)") } - let localURL = destinationRoot.appendingPathComponent(file.localPath) try fileManager.createDirectory( at: localURL.deletingLastPathComponent(), withIntermediateDirectories: true diff --git a/OSGKeyboardShared/Services/LocalASRPreferenceKeys.swift b/OSGKeyboardShared/Services/LocalASRPreferenceKeys.swift index 12bef80..6e5146e 100644 --- a/OSGKeyboardShared/Services/LocalASRPreferenceKeys.swift +++ b/OSGKeyboardShared/Services/LocalASRPreferenceKeys.swift @@ -5,4 +5,6 @@ import Foundation enum LocalASRPreferenceKeys { static let selectedModelId = "mac.localASR.selectedModelId" + /// Persists the user's preferred model download mirror (see `LocalASRDownloadSourcePreference`). + static let downloadSource = "mac.localASR.downloadSource" } diff --git a/OSGKeyboardShared/en.lproj/Shared.strings b/OSGKeyboardShared/en.lproj/Shared.strings index 0b240aa..8f6e208 100644 --- a/OSGKeyboardShared/en.lproj/Shared.strings +++ b/OSGKeyboardShared/en.lproj/Shared.strings @@ -164,6 +164,7 @@ "mac.overlay.polishing" = "Polishing"; "mac.overlay.live" = "Live"; "mac.overlay.done" = "Done"; +"mac.overlay.dragHint" = "Drag to move · double-click to reset"; "mac.record.start" = "Record"; "mac.record.stop" = "Stop"; "mac.record.pressStop" = "Press Stop"; @@ -305,7 +306,11 @@ "mac.error.qwen3LoadFailed" = "Failed to load Qwen3 model: %@"; "mac.error.qwen3InferenceFailed" = "Qwen3 transcription failed: %@"; "mac.localASR.models" = "Local ASR Engine & Models"; -"mac.localASR.modelsDesc" = "Models download directly. China uses ModelScope first; elsewhere Hugging Face first, then GitHub. Qwen3 1.7B downloads as multiple files."; +"mac.localASR.modelsDesc" = "Models download directly. Mainland China uses the hf-mirror mirror first; elsewhere the official Hugging Face endpoint first. Interrupted downloads resume automatically and fall back across mirrors."; +"mac.localASR.downloadSource" = "Download source"; +"mac.localASR.downloadSource.auto" = "Automatic (by region)"; +"mac.localASR.downloadSource.hfMirror" = "hf-mirror (recommended in China)"; +"mac.localASR.downloadSource.huggingface" = "Hugging Face (official)"; "mac.localASR.download" = "Download"; "mac.localASR.selectFolder" = "Choose folder"; "mac.localASR.openFolder" = "Open folder"; diff --git a/OSGKeyboardShared/zh-Hans.lproj/Shared.strings b/OSGKeyboardShared/zh-Hans.lproj/Shared.strings index 3b892d7..19b9308 100644 --- a/OSGKeyboardShared/zh-Hans.lproj/Shared.strings +++ b/OSGKeyboardShared/zh-Hans.lproj/Shared.strings @@ -164,6 +164,7 @@ "mac.overlay.polishing" = "润色中"; "mac.overlay.live" = "实时"; "mac.overlay.done" = "已完成"; +"mac.overlay.dragHint" = "拖动可移动位置,双击恢复默认"; "mac.record.start" = "开始录音"; "mac.record.stop" = "停止"; "mac.record.pressStop" = "点击停止"; @@ -305,7 +306,11 @@ "mac.error.qwen3LoadFailed" = "Qwen3 模型加载失败:%@"; "mac.error.qwen3InferenceFailed" = "Qwen3 转写失败:%@"; "mac.localASR.models" = "本地 ASR 引擎与模型"; -"mac.localASR.modelsDesc" = "模型可直接下载。中国大陆优先 ModelScope,其他地区优先 Hugging Face,再回退 GitHub。Qwen3 1.7B 以多文件方式下载。"; +"mac.localASR.modelsDesc" = "模型可直接下载。中国大陆优先 hf-mirror 镜像,其他地区优先 Hugging Face 官方;下载中断会自动断点续传并在镜像间回退。"; +"mac.localASR.downloadSource" = "下载源"; +"mac.localASR.downloadSource.auto" = "自动(按地区)"; +"mac.localASR.downloadSource.hfMirror" = "hf-mirror(中国大陆推荐)"; +"mac.localASR.downloadSource.huggingface" = "Hugging Face 官方"; "mac.localASR.download" = "下载"; "mac.localASR.selectFolder" = "选择目录"; "mac.localASR.openFolder" = "打开目录"; diff --git a/OSGKeyboardTests/LocalASRDownloadSourceSorterTests.swift b/OSGKeyboardTests/LocalASRDownloadSourceSorterTests.swift index 0376c1c..5518301 100644 --- a/OSGKeyboardTests/LocalASRDownloadSourceSorterTests.swift +++ b/OSGKeyboardTests/LocalASRDownloadSourceSorterTests.swift @@ -16,24 +16,43 @@ final class LocalASRDownloadSourceSorterTests: XCTestCase { ) } - func testChinaMainlandPrefersModelScope() { + func testChinaMainlandPrefersHFMirror() { let sources = [ source("github"), source("huggingface"), + source("hfmirror"), source("modelscope"), ] let sorted = LocalASRDownloadSourceSorter.sorted(sources, region: Locale.Region("CN")) - XCTAssertEqual(sorted.map(\.type), ["modelscope", "huggingface", "github"]) + XCTAssertEqual(sorted.map(\.type), ["hfmirror", "huggingface", "modelscope", "github"]) } func testGlobalPrefersHuggingFace() { let sources = [ source("github"), source("huggingface"), + source("hfmirror"), source("modelscope"), ] let sorted = LocalASRDownloadSourceSorter.sorted(sources, region: Locale.Region("US")) - XCTAssertEqual(sorted.map(\.type), ["huggingface", "github", "modelscope"]) + XCTAssertEqual(sorted.map(\.type), ["huggingface", "hfmirror", "github", "modelscope"]) + } + + func testUnknownRegionFallsBackToMirror() { + let sources = [source("huggingface"), source("hfmirror")] + let sorted = LocalASRDownloadSourceSorter.sorted(sources, region: nil) + XCTAssertEqual(sorted.map(\.type), ["hfmirror", "huggingface"]) + } + + func testManualPreferenceOverridesRegion() { + let sources = [source("hfmirror"), source("huggingface")] + // Even in China, an explicit HF preference wins. + let sorted = LocalASRDownloadSourceSorter.sorted( + sources, + region: Locale.Region("CN"), + preferred: .huggingface + ) + XCTAssertEqual(sorted.map(\.type), ["huggingface", "hfmirror"]) } func testSameTypeUsesPriority() { diff --git a/OSGKeyboardTests/LocalASRModelCatalogTests.swift b/OSGKeyboardTests/LocalASRModelCatalogTests.swift index 228c8ea..1be9207 100644 --- a/OSGKeyboardTests/LocalASRModelCatalogTests.swift +++ b/OSGKeyboardTests/LocalASRModelCatalogTests.swift @@ -9,54 +9,62 @@ final class LocalASRModelCatalogTests: XCTestCase { func testBundledCatalogLoads() throws { let catalog = try LocalASRModelCatalog.loadBundled() XCTAssertEqual(catalog.schemaVersion, 1) - XCTAssertEqual(catalog.defaultModelId, "sherpa-qwen3-0.6b-int8") - XCTAssertFalse(catalog.models.contains { $0.id == "qwen3-mlx-1.7b" }) - XCTAssertTrue(catalog.models.contains { $0.id == "sherpa-qwen3-0.6b-int8" }) - XCTAssertTrue(catalog.models.contains { $0.id == "sherpa-qwen3-1.7b-int8" }) - XCTAssertTrue(catalog.models.contains { $0.id == "sherpa-sensevoice-small-int8" }) - XCTAssertFalse(catalog.models.contains { $0.id == "sherpa-paraformer-zh-int8" }) + XCTAssertEqual(catalog.defaultModelId, "qwen3-mlx-0.6b-4bit") + XCTAssertTrue(catalog.models.contains { $0.id == "qwen3-mlx-0.6b-4bit" }) + XCTAssertTrue(catalog.models.contains { $0.id == "qwen3-mlx-1.7b-4bit" }) + XCTAssertFalse(catalog.models.contains { $0.id == "sherpa-qwen3-0.6b-int8" }) + XCTAssertTrue(catalog.runtimes.isEmpty) XCTAssertEqual( - LocalASRModelCatalog.model("sherpa-sensevoice-small-int8", in: catalog)?.badgeKey, - "mac.localASR.badge.fastest" - ) - XCTAssertEqual( - LocalASRModelCatalog.model("sherpa-qwen3-0.6b-int8", in: catalog)?.badgeKey, + LocalASRModelCatalog.model("qwen3-mlx-0.6b-4bit", in: catalog)?.badgeKey, "mac.localASR.badge.balanced" ) XCTAssertEqual( - LocalASRModelCatalog.model("sherpa-qwen3-1.7b-int8", in: catalog)?.badgeKey, + LocalASRModelCatalog.model("qwen3-mlx-1.7b-4bit", in: catalog)?.badgeKey, "mac.localASR.badge.quality" ) } - func testSherpaQwen317BUsesRepositoryInstall() throws { + func testMLX06BUsesRepositoryInstall() throws { let catalog = try LocalASRModelCatalog.loadBundled() - let model = try XCTUnwrap(LocalASRModelCatalog.model("sherpa-qwen3-1.7b-int8", in: catalog)) + let model = try XCTUnwrap(LocalASRModelCatalog.model("qwen3-mlx-0.6b-4bit", in: catalog)) XCTAssertEqual(model.installKind, .repository) - XCTAssertTrue(model.sources?.contains(where: { $0.type == "modelscope" && $0.isRepository }) == true) + XCTAssertEqual(model.backend, .mlx) XCTAssertTrue(model.sources?.contains(where: { $0.type == "huggingface" && $0.isRepository }) == true) } - func testCapabilitiesForSherpaQwen3() throws { + func testMLXSourcesIncludeHFMirrorAndOfficial() throws { let catalog = try LocalASRModelCatalog.loadBundled() - let model = try XCTUnwrap(LocalASRModelCatalog.model("sherpa-qwen3-0.6b-int8", in: catalog)) - let caps = LocalASRModelCatalog.capabilities(for: model) - XCTAssertEqual(caps.hotwordMode, .recognizerScoped) - XCTAssertTrue(model.supportsHotwords) + for id in ["qwen3-mlx-0.6b-4bit", "qwen3-mlx-1.7b-4bit"] { + let model = try XCTUnwrap(LocalASRModelCatalog.model(id, in: catalog)) + let types = Set(model.sources?.map(\.type) ?? []) + XCTAssertTrue(types.contains("hfmirror"), "\(id) should offer the hf-mirror source") + XCTAssertTrue(types.contains("huggingface"), "\(id) should offer the official HF source") + XCTAssertFalse(types.contains("modelscope"), "\(id) should drop the dead ModelScope link") + + let mirror = try XCTUnwrap(model.sources?.first { $0.type == "hfmirror" }) + XCTAssertTrue(mirror.baseURL?.hasPrefix("https://hf-mirror.com/") == true) + let remoteFiles = Set(mirror.files?.map(\.remotePath) ?? []) + XCTAssertTrue(remoteFiles.contains("model.safetensors")) + XCTAssertTrue(remoteFiles.contains("preprocessor_config.json")) + // Files that don't exist in the real repo must not be listed. + XCTAssertFalse(remoteFiles.contains("tokenizer.json")) + XCTAssertFalse(remoteFiles.contains("special_tokens_map.json")) + } } - func testCapabilitiesForSenseVoice() throws { + func testCapabilitiesForMLXQwen3() throws { let catalog = try LocalASRModelCatalog.loadBundled() - let model = try XCTUnwrap(LocalASRModelCatalog.model("sherpa-sensevoice-small-int8", in: catalog)) + let model = try XCTUnwrap(LocalASRModelCatalog.model("qwen3-mlx-0.6b-4bit", in: catalog)) let caps = LocalASRModelCatalog.capabilities(for: model) - XCTAssertEqual(caps.hotwordMode, .none) - XCTAssertFalse(model.supportsHotwords) + XCTAssertEqual(caps.hotwordMode, .promptOnly) + XCTAssertTrue(caps.supportsStreaming) + XCTAssertTrue(model.supportsHotwords) } func testManifestRoundTrip() throws { let manifest = LocalASRInstalledManifest( - selectedModelId: "sherpa-qwen3-0.6b-int8", - installedModelIDs: ["sherpa-qwen3-0.6b-int8"] + selectedModelId: "qwen3-mlx-0.6b-4bit", + installedModelIDs: ["qwen3-mlx-0.6b-4bit"] ) let url = FileManager.default.temporaryDirectory .appendingPathComponent("manifest-\(UUID().uuidString).json") @@ -76,7 +84,7 @@ final class LocalASRModelCatalogTests: XCTestCase { func testBiasDiagnosticsStoreRoundTrip() { LocalASRBiasDiagnosticsStore.clear() let payload = LocalASRBiasPayload( - hardHotwords: ["Cursor"], + hardHotwords: [], promptBias: "test", corpusContext: nil, polishFragment: "fragment", @@ -85,17 +93,17 @@ final class LocalASRModelCatalogTests: XCTestCase { ) LocalASRBiasDiagnosticsStore.save( payload: payload, - modelId: "sherpa-qwen3-0.6b-int8", - backendLabel: "Sherpa Qwen3" + modelId: "qwen3-mlx-0.6b-4bit", + backendLabel: "Qwen3-ASR 0.6B" ) let snapshot = LocalASRBiasDiagnosticsStore.load() - XCTAssertEqual(snapshot?.modelId, "sherpa-qwen3-0.6b-int8") + XCTAssertEqual(snapshot?.modelId, "qwen3-mlx-0.6b-4bit") XCTAssertEqual(snapshot?.diagnostics.userTermCount, 2) - XCTAssertEqual(snapshot?.hotwordCount, 1) + XCTAssertEqual(snapshot?.hotwordCount, 0) LocalASRBiasDiagnosticsStore.clear() } - func testSherpaAdapterProducesHardHotwords() throws { + func testMLXAdapterProducesPromptBiasNotHardHotwords() throws { let fixtureURL = FileManager.default.temporaryDirectory .appendingPathComponent("phrases-\(UUID().uuidString).tsv") try "word\tpinyin\tsource\tweight\nSwiftUI\tswift ui\tcomputer_terms\t5\n" @@ -109,11 +117,37 @@ final class LocalASRModelCatalogTests: XCTestCase { LocalASRBiasRequest( dictionary: dict, locale: Locale(identifier: "zh-CN"), - capabilities: .sherpaQwen3 + capabilities: .qwen3MLX ), lexicon: BuiltinLexiconIndex(fixtureURL: fixtureURL) ) - XCTAssertFalse(payload.hardHotwords.isEmpty) - XCTAssertTrue(payload.hardHotwords.contains("Kubernetes")) + XCTAssertTrue(payload.hardHotwords.isEmpty) + XCTAssertNotNil(payload.promptBias) + XCTAssertTrue(payload.promptBias?.contains("Kubernetes") == true) + } + + func testLegacySherpaModelIdMapsToMLXDefault() { + let legacyIds = [ + "sherpa-qwen3-0.6b-int8", + "sherpa-qwen3-1.7b-int8", + "sherpa-sensevoice-small-int8", + ] + for id in legacyIds { + XCTAssertEqual(migrateLegacyModelId(id), "qwen3-mlx-0.6b-4bit") + } + XCTAssertEqual(migrateLegacyModelId("qwen3-mlx-1.7b-4bit"), "qwen3-mlx-1.7b-4bit") + } + + private func migrateLegacyModelId(_ id: String) -> String { + switch id { + case "sherpa-qwen3-0.6b-int8", + "sherpa-qwen3-1.7b-int8", + "sherpa-sensevoice-small-int8", + "sherpa-paraformer-zh-int8", + "qwen3-mlx-1.7b": + return "qwen3-mlx-0.6b-4bit" + default: + return id + } } } diff --git a/Scripts/ensure-mlx-audio-swift.sh b/Scripts/ensure-mlx-audio-swift.sh new file mode 100755 index 0000000..e206016 --- /dev/null +++ b/Scripts/ensure-mlx-audio-swift.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Ensures vendored mlx-audio-swift is present and patched for StreamingConfig.context. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +VENDOR="$ROOT/ThirdParty/mlx-audio-swift" +PATCH_MARKER="$VENDOR/.osg-context-patch-applied" + +if [[ ! -f "$VENDOR/Package.swift" ]]; then + echo "Cloning mlx-audio-swift into ThirdParty/..." + git clone --depth 1 https://github.com/Blaizzy/mlx-audio-swift "$VENDOR" +fi + +if [[ -f "$PATCH_MARKER" ]]; then + echo "mlx-audio-swift context patch already applied." + exit 0 +fi + +python3 <<'PY' +from pathlib import Path + +root = Path("ThirdParty/mlx-audio-swift") +manifest = root / "Package.swift" +types = root / "Sources/MLXAudioSTT/Streaming/StreamingTypes.swift" +session = root / "Sources/MLXAudioSTT/Streaming/StreamingInferenceSession.swift" + +# Pin mlx-swift to 0.31.3: 0.31.4+ adds an unconditional swift-argument-parser +# dependency (for its `encuda` executable) that breaks offline resolution here. +mtext = manifest.read_text() +if "mlx-swift.git\", exact:" not in mtext: + mtext = mtext.replace( + '.package(url: "https://github.com/ml-explore/mlx-swift.git", .upToNextMajor(from: "0.30.6")),', + '.package(url: "https://github.com/ml-explore/mlx-swift.git", exact: "0.31.3"),', + ) + manifest.write_text(mtext) + +text = types.read_text() +if "public var context: String?" not in text: + text = text.replace( + " public var language: String?\n", + " public var language: String?\n" + " /// Optional soft-prompt context (vocabulary / domain hints) injected into the system turn.\n" + " public var context: String?\n", + ) + text = text.replace( + " language: String? = \"English\",\n temperature: Float = 0.0,", + " language: String? = \"English\",\n context: String? = nil,\n temperature: Float = 0.0,", + ) + text = text.replace( + " self.language = language\n self.temperature = temperature", + " self.language = language\n self.context = context\n self.temperature = temperature", + ) + types.write_text(text) + +text = session.read_text() +text = text.replace( + " language: params.config.language\n )", + " context: params.config.context ?? \"\",\n language: params.config.language\n )", +) +text = text.replace( + " language: config.language\n )", + " context: config.context ?? \"\",\n language: config.language\n )", +) +session.write_text(text) +PY + +touch "$PATCH_MARKER" +echo "Applied mlx-audio-swift context patch." diff --git a/Scripts/generate-xcodeproj.sh b/Scripts/generate-xcodeproj.sh index 3667978..40b422f 100755 --- a/Scripts/generate-xcodeproj.sh +++ b/Scripts/generate-xcodeproj.sh @@ -35,8 +35,12 @@ if [[ ! -f "$PRECONFIG_LOCAL" ]]; then echo "Edit deepseek in that file before using the local engine's built-in polish." fi +"$ROOT/Scripts/ensure-mlx-audio-swift.sh" + xcodegen generate +"$ROOT/Scripts/patch-spm-local-package.sh" + if python3 - "$PBXPROJ" <<'PY' import re import sys diff --git a/Scripts/patch-spm-local-package.sh b/Scripts/patch-spm-local-package.sh new file mode 100755 index 0000000..3e72d96 --- /dev/null +++ b/Scripts/patch-spm-local-package.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# XcodeGen 2.43 omits `package = …` on XCSwiftPackageProductDependency for local +# path packages, which makes Xcode show "Missing package product 'MLXAudioSTT'". +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PBXPROJ="$ROOT/OSGKeyboard.xcodeproj/project.pbxproj" + +if [[ ! -f "$PBXPROJ" ]]; then + echo "patch-spm-local-package: $PBXPROJ not found — run xcodegen first." >&2 + exit 1 +fi + +python3 - "$PBXPROJ" <<'PY' +import re +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +text = path.read_text() + +package_ref = re.search( + r'(\w+) /\* XCLocalSwiftPackageReference "ThirdParty/mlx-audio-swift" \*/ = \{\n' + r'\s*isa = XCLocalSwiftPackageReference;\n' + r'\s*relativePath = "ThirdParty/mlx-audio-swift";\n' + r'\s*\};', + text, +) +if not package_ref: + print("patch-spm-local-package: local package reference not found — skip") + sys.exit(0) + +package_id = package_ref.group(1) +product_block = re.search( + r'(\w+) /\* MLXAudioSTT \*/ = \{\n' + r'\s*isa = XCSwiftPackageProductDependency;\n' + r'(\s*productName = MLXAudioSTT;\n)' + r'\s*\};', + text, +) +if not product_block: + print("patch-spm-local-package: MLXAudioSTT product dependency not found — skip") + sys.exit(0) + +product_id = product_block.group(1) +if f"package = {package_id}" in text: + print("patch-spm-local-package: already patched") + sys.exit(0) + +replacement = ( + f"{product_id} /* MLXAudioSTT */ = {{\n" + f"\t\t\tisa = XCSwiftPackageProductDependency;\n" + f"\t\t\tpackage = {package_id} /* XCLocalSwiftPackageReference \"ThirdParty/mlx-audio-swift\" */;\n" + f"\t\t\tproductName = MLXAudioSTT;\n" + f"\t\t}};" +) +text = text[: product_block.start()] + replacement + text[product_block.end() :] +path.write_text(text) +print("patch-spm-local-package: linked MLXAudioSTT → ThirdParty/mlx-audio-swift") +PY diff --git a/ThirdParty/README.md b/ThirdParty/README.md new file mode 100644 index 0000000..06d0722 --- /dev/null +++ b/ThirdParty/README.md @@ -0,0 +1,15 @@ +# ThirdParty + +## mlx-audio-swift + +macOS local ASR links `MLXAudioSTT` from [Blaizzy/mlx-audio-swift](https://github.com/Blaizzy/mlx-audio-swift). + +Before `xcodegen generate` or opening the project, run: + +```bash +./Scripts/ensure-mlx-audio-swift.sh +``` + +This clones the package (if missing) and applies OSG's `StreamingConfig.context` patch for vocabulary prompts. + +The clone lives at `ThirdParty/mlx-audio-swift/` (gitignored). SPM resolves mlx-audio-swift's own dependencies on first Mac build. diff --git a/docs/local-asr-architecture.md b/docs/local-asr-architecture.md index efe8e29..34b6668 100644 --- a/docs/local-asr-architecture.md +++ b/docs/local-asr-architecture.md @@ -2,7 +2,7 @@ > **文档状态**:架构规划(非实现规格) > **适用范围**:macOS 本地听写;与 iOS 键盘扩展、云 ASR 路径的关系见各节说明。 -> **核心结论**:短期不换主模型,优先打通 **词库感知管道**;中期用 POC 验证 **Sherpa Qwen3 hard hotwords** 是否值得成为热词主线。 +> **核心结论**:macOS 本地听写默认 **Qwen3 MLX 真流式**(mlx-audio-swift);词库经 `LocalASRBiasAdapter` 以 soft prompt + 后处理注入。Sherpa offline 已移除。 --- diff --git a/docs/mac-qwen3-mlx-streaming-migration-plan.md b/docs/mac-qwen3-mlx-streaming-migration-plan.md index 1f46cfe..fa51be6 100644 --- a/docs/mac-qwen3-mlx-streaming-migration-plan.md +++ b/docs/mac-qwen3-mlx-streaming-migration-plan.md @@ -1,6 +1,6 @@ # Mac 本地 ASR 迁移计划:Whisperer 流式 + 词库分层 -> **文档状态**:实施计划(待评审) +> **文档状态**:实施计划(**已评审,决策已冻结**) > **适用范围**:macOS 本地听写(`OSGKeyboardMac`) > **关联文档**:[`local-asr-architecture.md`](./local-asr-architecture.md) > **创建日期**:2026-07-14 @@ -26,7 +26,9 @@ | **模型** | Qwen3-ASR 0.6B 4-bit(默认)/ 1.7B 4-bit(高质量档) | | **热词模式** | `promptOnly`(`StreamingConfig.context`),**禁用** Sherpa hard hotwords | | **词库适配** | 复用现有 `LocalASRBiasAdapter`,不新建平行词库系统 | -| **废弃** | Sherpa Qwen3 offline 子进程 + `ChunkedUtterancePipeline` 本地主线 | +| **废弃** | Sherpa Qwen3 offline 子进程 + `ChunkedUtterancePipeline` 本地主线(**catalog 直接移除**,不保留 advanced) | +| **partial UI** | **仅 overlay 预览**;松开后润色完成再注入前台 App | +| **默认模型** | 0.6B 4-bit 默认;1.7B 可选下载(设置 UI 已有) | ### 1.3 非目标(本期不做) @@ -204,7 +206,7 @@ Sherpa / Apple Speech 保留为 **fallback provider**,不作为默认。 - 新增条目:`qwen3-mlx-0.6b-4bit`、`qwen3-mlx-1.7b-4bit` - `backend: mlx`,`hotwordMode: promptOnly` - 默认模型 ID 从 `sherpa-qwen3-0.6b-int8` 改为 `qwen3-mlx-0.6b-4bit` - - 保留 Sherpa 条目但标记 `deprecated` / 高级选项 + - **移除**所有 Sherpa 模型与 runtime 条目(不保留 advanced / fallback) 4. **新文件骨架** - `OSGKeyboardMac/MacMLXStreamingASRProvider.swift` @@ -239,8 +241,8 @@ Sherpa / Apple Speech 保留为 **fallback provider**,不作为默认。 - 删除/绕开 `liveCaptureTask` + `ChunkedUtterancePipeline` 本地路径 - 学 Whisperer `AppState`: - 按下:创建 session + 100ms `feedAudio` timer - - 监听 `session.events` → 更新 `transcript` / `isStreamingPartial` - - 松开:进入 Phase 3 tail drain + - 监听 `session.events` → 更新 overlay `transcript` / `isStreamingPartial`(**仅预览,不插入前台 App**) + - 松开:进入 Phase 3 tail drain → 润色 → 再注入 3. **`MacDictationPipeline.resolveLocalBias`** - 已有实现保留;capabilities 改为 `.qwen3MLX` @@ -331,10 +333,11 @@ Sherpa / Apple Speech 保留为 **fallback provider**,不作为默认。 qwen3-mlx 失败 → Apple Speech(现有 MacSpeechLocalASR) 模型缺失 → 引导下载 / 云模式 ``` + (无 Sherpa 回退) -4. **废弃路径标记** - - `MacSherpaONNXRunner` / `MacSherpaLocalASR` 保留但默认隐藏 - - `MacLocalASRChunkAdapter` 仅 cloud chunked 或 legacy flag 使用 +4. **Sherpa 代码清理** + - 删除 `MacSherpaONNXRunner` / `MacSherpaLocalASR` / Sherpa runtime 下载逻辑 + - `MacLocalASRChunkAdapter` 仅保留 cloud chunked 路径(若仍需要) **验收** @@ -385,13 +388,15 @@ Sherpa / Apple Speech 保留为 **fallback provider**,不作为默认。 | `OSGKeyboardMac/MacLocalASRModelSettingsView.swift` | 模型档 + diagnostics | | `docs/local-asr-architecture.md` | 与本文对齐 | -### 5.3 废弃(保留代码,默认不启用) +### 5.3 删除(Sherpa 路径) | 文件 | 说明 | |------|------| -| `OSGKeyboardMac/MacSherpaONNXRunner.swift` | hard hotwords 路径 | -| `OSGKeyboardMac/MacLocalASRChunkAdapter.swift` | Sherpa chunked | -| `OSGKeyboardShared/.../ChunkedUtterancePipeline.swift` | Mac 本地不再使用 | +| `OSGKeyboardMac/MacSherpaONNXRunner.swift` | 移除 | +| `OSGKeyboardMac/MacSherpaLocalASR.swift` | 移除 | +| `local-asr-catalog.json` 中 Sherpa runtime / 模型条目 | 移除 | +| `MacLocalASRChunkAdapter.swift`(Sherpa 专用部分) | 移除或仅留 cloud | +| Mac 本地 `ChunkedUtterancePipeline` 调用 | 移除 | --- @@ -488,10 +493,10 @@ Sherpa / Apple Speech 保留为 **fallback provider**,不作为默认。 ## 9. 回滚策略 -1. **Feature flag**:`mac.localASR.backend = mlxQwen3 | sherpaQwen3 | appleSpeech` -2. **模型级回滚**:catalog 默认指回 Sherpa(不推荐长期使用) -3. **云模式**:本地失败自动提示切换云 ASR(现有路径) -4. **词库不受影响**:adapter 层与引擎解耦,回滚不改词库 +1. **Feature flag**:`mac.localASR.backend = mlxQwen3 | appleSpeech | cloud` +2. **云模式**:本地 MLX / Apple Speech 失败时提示切换云 ASR +3. **词库不受影响**:adapter 层与引擎解耦 +4. **无 Sherpa 回滚**:已决策直接移除 --- @@ -536,13 +541,48 @@ Phase 6 评测 / 灰度 / 发布 ─────────────── --- -## 13. 开放问题(评审时确认) +## 13. 已冻结决策(2026-07-14 评审) -1. **mlx-audio-swift 依赖方式**:直接 pin `main` vs fork 带 `context` patch? -2. **partial UI vs 增量插入**:Mac 默认仅 overlay 预览,还是像 Whisperer 边说边插入? -3. **0.6B vs 1.7B 默认**:质量优先还是延迟优先? -4. **Sherpa 条目何时从 catalog 移除**:灰度后一个版本 vs 长期保留 advanced? -5. **是否 Phase 4 引入 Silero VAD**:或 RMS 门控足够? +| # | 问题 | 决策 | +|---|------|------| +| 1 | mlx-audio-swift 引入方式 | **Mac target 通过 SPM 直接引入** `Blaizzy/mlx-audio-swift`;iOS 仍零 SPM。若 upstream streaming 缺 `context`,在 OSG fork 打小 patch 后 pin revision(见 §13.1) | +| 2 | partial UI | **仅 overlay 预览**;松开后经润色链再注入(与云路径一致) | +| 3 | 默认模型 | **0.6B 默认**;1.7B 可选下载;现有设置 UI 复用 | +| 4 | Sherpa | **直接移除**(catalog + 代码 + runtime 下载),不保留 advanced | +| 5 | 静音检测 | **Phase 4 先上轻量 RMS 门控**;Silero VAD 仅当 RMS 实测不够再评估(见 §13.2) | + +### 13.1 依赖引入说明(给开发) + +「可以直接引入」= 在 `project.yml` 为 **仅 `OSGKeyboardMac` target** 添加 Swift Package: + +```yaml +packages: + MLXAudio: + url: https://github.com/Blaizzy/mlx-audio-swift + from: "0.1.0" # 或 pin 到具体 revision / OSG fork + +targets: + OSGKeyboardMac: + dependencies: + - package: MLXAudio + product: MLXAudioSTT +``` + +- **不需要**把整个仓库 copy 进 OSGKeyboard 源码树(除非 fork patch 暂无法 upstream)。 +- **需要** macOS + Xcode 16+ 本机构建;`xcodegen generate` 后 Xcode 会拉取并编译 MLX。 +- **模型权重**仍走现有 `LocalASRModelManager` 下载管线,不随 SPM 打包进 app。 +- **唯一前置条件**:确认 streaming API 支持 `context`(词库 prompt);若无,fork 加 2 处 `buildPrompt` 改动即可。 + +### 13.2 静音检测说明(给产品) + +把「用户是否在说话」想象成两道筛子: + +| 方案 | 产品语言 | 优点 | 缺点 | +|------|----------|------|------| +| **RMS 门控**(先做) | 听音量大小:太安静就不送给识别引擎 | 实现简单、几乎不增加包体、不拖慢首字 | 嘈杂环境可能把背景声当「在说话」 | +| **Silero VAD**(备选) | 专门训练过的「人声探测器」,区分人声 vs 键盘/风扇声 | 静音误触发更少 | 多一个模型依赖、开发和评测成本更高 | + +**决策**:先用 RMS(成本低、能解决「按住不说话却喷词」的主痛点);若内测发现办公室/咖啡厅误触发仍多,再加 Silero。 --- diff --git a/project.yml b/project.yml index 89b06dd..78f2b1c 100644 --- a/project.yml +++ b/project.yml @@ -22,12 +22,16 @@ options: # v0.2.0: dropped the local `Qwen3Speech` SPM fork. The "local" engine # now uses iOS 26 `SpeechAnalyzer` + `DictationTranscriber` exclusively, -# which keeps the dependency surface at zero SPM packages / Pods / +# which keeps the iOS dependency surface at zero SPM packages / Pods / # Carthage (matches the long-standing "zero dependencies" promise in the # README). Optional post-ASR cloud polish routes through DeepSeek (or # any OpenAI-compatible endpoint) using the existing `LLMClient`. -# iOS targets remain zero-SPM. macOS local ASR downloads the sherpa-onnx -# runtime on demand instead of linking SwiftPM speech packages. +# iOS targets remain zero-SPM-linked. macOS local ASR links mlx-audio-swift +# (ThirdParty/) for Qwen3 MLX streaming; weights download via catalog. + +packages: + MLXAudio: + path: ThirdParty/mlx-audio-swift settings: base: @@ -393,7 +397,7 @@ targets: # Reuses the platform-agnostic core files from OSGKeyboardShared at the # source level (no framework), excluding the iOS-only files that import # SpeechAnalyzer / AVAudioSession / UIKit / SwiftUI views. Local mode uses - # a downloaded sherpa-onnx runtime plus catalog-managed ONNX models. + # Qwen3 MLX streaming via mlx-audio-swift plus catalog-managed weights. OSGKeyboardMac: type: application platform: macOS @@ -488,6 +492,8 @@ targets: # Required for Developer ID distribution + notarization (outside App Store). ENABLE_HARDENED_RUNTIME: YES dependencies: + - package: MLXAudio + product: MLXAudioSTT - sdk: Speech.framework - sdk: AVFoundation.framework - sdk: Charts.framework