feat: macOS architecture, cloud ASR/LLM providers, and 6-step iOS onboarding

- Add macOS menu-bar dictation app with local ASR models (SenseVoice/Qwen3),
  global Option hotkey, and bottom overlay
- Add cloud ASR/LLM providers (Anthropic, Volcengine, Bailian, and more) with
  provider logos, model listing, and connection checks
- Add shared 7-day usage stats UI (UsageStatsCluster / SevenDayUsageChart)
- Add iOS onboarding step 6 for polish LLM setup; hide custom-language-model
  diagnostic toggle behind DEBUG
- Unify iOS onboarding tagline with the macOS brand line ("开口即文字。")
- Rewrite README (Chinese-first, product-oriented) and refresh GitHub Pages
This commit is contained in:
Rocky
2026-07-11 19:10:20 +08:00
parent cdf833935a
commit cc8dd1070a
116 changed files with 6659 additions and 2634 deletions
@@ -105,6 +105,12 @@ public final class KeyboardViewController: UIInputViewController {
super.viewDidAppear(animated)
disableSystemGestureDelays()
keyboardHeightConstraint?.constant = targetKeyboardHeight
refreshReturnKeyRole()
}
public override func textDidChange(_ textInput: UITextInput?) {
super.textDidChange(textInput)
refreshReturnKeyRole()
}
public override var preferredScreenEdgesDeferringSystemGestures: UIRectEdge {
@@ -195,6 +201,21 @@ public final class KeyboardViewController: UIInputViewController {
}
}
private func refreshReturnKeyRole() {
state.returnKeyRole = returnKeyRole(for: textDocumentProxy.returnKeyType ?? .default)
}
private func returnKeyRole(for returnKeyType: UIReturnKeyType) -> State.ReturnKeyRole {
switch returnKeyType {
case .send, .go, .search, .join, .route, .google, .yahoo, .continue, .emergencyCall:
return .send
case .default, .next, .done:
return .newline
@unknown default:
return .newline
}
}
// MARK: - System keyboard chrome
private func configureDictationBehavior() {
@@ -48,8 +48,21 @@ final class KeyboardFlowCoordinator {
/// `adoptHostBusyStateIfNeeded` from re-entering `.processing` after a
/// stale App Group snapshot still says `reason=processing`.
private var lastConsumedUtteranceId: UUID?
/// Utterance we just asked the host to stop. Until the host publishes
/// processing/final state, stale App Group snapshots can still say
/// `reason=recording`; do not re-adopt that utterance as locally active.
private var lastStoppedUtteranceId: UUID?
private var currentCommandSeq: Int64 = 0
private var lastAvailabilityTraceSignature = ""
/// When true, `completeFlowStartHandoff` starts recording after the host
/// publishes ready set only for an explicit mic press.
private var recordAfterHandoff = false
/// When true, `startHostReadyWaitIfNeeded` starts recording once ready
/// (mic pressed while session was still warming / mid ready-flap).
private var recordWhenHostReady = false
/// Ignores single-frame "host dead" samples before allowing a cold-start jump
/// from non-press recovery paths.
private var coldStartDebouncer = FlowColdStartDebouncer()
init(
state: KeyboardState,
@@ -120,6 +133,8 @@ final class KeyboardFlowCoordinator {
recomputeMicVoiceAvailability()
startHostReadyWaitIfNeeded()
// Proactive host auto-launch is disabled (FlowHandoffPolicy): a single
// stale ready snapshot after finalize must never open startflow.
// Only surface "session ended" when the session contract *genuinely*
// dropped (expired / cleared). A transient host-ready flap engine
@@ -221,6 +236,7 @@ final class KeyboardFlowCoordinator {
// forever miss the real delivery and leaves the mic white forever.
guard let busyId = snapshot.busyUtteranceId else { return }
guard busyId != lastConsumedUtteranceId else { return }
guard busyId != lastStoppedUtteranceId else { return }
activeSessionId = sessionId
currentUtteranceId = busyId
isPendingFlowStart = false
@@ -268,6 +284,7 @@ final class KeyboardFlowCoordinator {
state.lastTranscript = ""
stopFlowWatchdog()
currentUtteranceId = nil
lastStoppedUtteranceId = nil
traceState(
"stickyProcessing.cleared",
extra: hostReady ? "hostReady=1" : "hostReady=0"
@@ -280,16 +297,24 @@ final class KeyboardFlowCoordinator {
guard !isPendingFlowStart else { return }
guard FlowSessionBridge.isSessionActive() else {
stopHostReadyWait()
if recordWhenHostReady {
// Session gone while waiting escalate to a real cold start.
let shouldRecord = recordWhenHostReady
recordWhenHostReady = false
beginFlowStart(recordAfterHandoff: shouldRecord)
}
return
}
// Host busy waiting for ready. Do not spin the ready-wait poll.
if let reason = FlowSessionBridge.readySnapshot()?.reason,
reason == .recording || reason == .processing {
stopHostReadyWait()
recordWhenHostReady = false
return
}
guard !FlowSessionBridge.isHostReady() else {
if FlowSessionBridge.isHostReady() {
stopHostReadyWait()
finishHostReadyWaitIfNeeded()
return
}
@@ -300,16 +325,50 @@ final class KeyboardFlowCoordinator {
guard let self, !Task.isCancelled else { return }
FlowSessionBridge.reloadFromDisk()
self.recomputeMicVoiceAvailability()
if self.state.micVoiceAvailability.isReady
|| self.state.micVoiceAvailability == .recording
if self.state.micVoiceAvailability.isReady {
self.finishHostReadyWaitIfNeeded()
return
}
if self.state.micVoiceAvailability == .recording
|| self.state.micVoiceAvailability == .processing {
self.recordWhenHostReady = false
return
}
// Host died mid-wait only cold-start after debounced dead samples.
let dead = FlowHandoffPolicy.shouldOpenHostColdStart(
sessionActive: FlowSessionBridge.isSessionActive(),
hostReachable: FlowSessionBridge.isHostReachable(),
hostStale: FlowSessionBridge.isHostStale(),
withinReadyGrace: false
)
if self.coldStartDebouncer.observe(hostTrulyDead: dead) {
let shouldRecord = self.recordWhenHostReady
self.recordWhenHostReady = false
self.coldStartDebouncer.reset()
self.beginFlowStart(recordAfterHandoff: shouldRecord)
return
}
try? await Task.sleep(nanoseconds: 150_000_000)
}
// Timed out still not ready if the user asked to record, cold-start.
guard let self else { return }
if self.recordWhenHostReady {
let shouldRecord = self.recordWhenHostReady
self.recordWhenHostReady = false
self.beginFlowStart(recordAfterHandoff: shouldRecord)
}
}
}
private func finishHostReadyWaitIfNeeded() {
coldStartDebouncer.reset()
guard recordWhenHostReady else { return }
recordWhenHostReady = false
guard state.micVoiceAvailability.isReady else { return }
startFlowRecording()
traceState("hostReadyWait.recordStarted")
}
private func stopHostReadyWait() {
hostReadyWaitTask?.cancel()
hostReadyWaitTask = nil
@@ -338,9 +397,6 @@ final class KeyboardFlowCoordinator {
recomputeMicVoiceAvailability()
switch state.micVoiceAvailability {
case .ready:
detectAndStoreAppContext()
startFlowRecording()
case .unavailable(.missingAPIKey):
return
case .unavailable(.noFullAccess):
@@ -348,18 +404,43 @@ final class KeyboardFlowCoordinator {
state.phase = .error(.fullAccessRequired, message: msg)
scheduleAutoClearError()
recomputeMicVoiceAvailability()
return
case .unavailable(.appGroupUnavailable):
let msg = ExtL10n.string("keyboard.error.appGroupCommunication")
state.phase = .error(.appGroupUnavailable, message: msg)
scheduleAutoClearError()
recomputeMicVoiceAvailability()
case .unavailable(.preparingSession):
return
default:
break
}
let withinReadyGrace = lastHostReadyAt > 0
&& (Date().timeIntervalSince1970 - lastHostReadyAt) <= Self.hostReadyGrace
let action = FlowHandoffPolicy.micPressAction(
availability: state.micVoiceAvailability,
sessionActive: FlowSessionBridge.isSessionActive(),
hostReachable: FlowSessionBridge.isHostReachable(),
hostStale: FlowSessionBridge.isHostStale(),
withinReadyGrace: withinReadyGrace
)
switch action {
case .startRecording:
detectAndStoreAppContext()
beginFlowStart()
case .unavailable(.hostNotReady):
startFlowRecording()
case .waitForHostReady(let recordWhenReady):
detectAndStoreAppContext()
beginFlowStart()
case .recording, .processing:
recordWhenHostReady = recordWhenReady
coldStartDebouncer.reset()
startHostReadyWaitIfNeeded()
traceState(
"pressBegan.waitForHostReady",
extra: recordWhenReady ? "recordWhenReady=1" : "recordWhenReady=0"
)
case .openHostColdStart:
detectAndStoreAppContext()
beginFlowStart(recordAfterHandoff: true)
case .ignore:
return
}
}
@@ -374,19 +455,23 @@ final class KeyboardFlowCoordinator {
isFlowRecording = false
stopUtteranceCountdown()
ExtensionScreenWakeLock.release()
lastStoppedUtteranceId = currentUtteranceId
writeCommand(.stopRecording)
debug("pressEnded wrote stop command")
state.phase = .processing
state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
recomputeMicVoiceAvailability()
startFlowResultWatchdog()
recomputeMicVoiceAvailability()
}
func beginFlowStart() {
func beginFlowStart(recordAfterHandoff: Bool = false) {
guard !isPendingFlowStart else {
traceState("beginFlowStart.ignored", extra: "reason=pendingAlreadyTrue")
return
}
self.recordAfterHandoff = recordAfterHandoff
recordWhenHostReady = false
coldStartDebouncer.reset()
isPendingFlowStart = true
isFlowRecording = false
flowStartDeadline = Date().timeIntervalSince1970 + FlowWatchdog.startTimeout
@@ -394,7 +479,10 @@ final class KeyboardFlowCoordinator {
recomputeMicVoiceAvailability()
openHostApp("startflow")
startFlowStartWatchdog()
traceState("beginFlowStart.started")
traceState(
"beginFlowStart.started",
extra: recordAfterHandoff ? "recordAfterHandoff=1" : "recordAfterHandoff=0"
)
}
func handleHostAppOpenResult(path: String, success: Bool) {
@@ -406,6 +494,7 @@ final class KeyboardFlowCoordinator {
// guide the user to open OSGKeyboard manually.
if path == "startflow", isPendingFlowStart {
isPendingFlowStart = false
recordAfterHandoff = false
flowStartDeadline = 0
stopFlowWatchdog()
traceState("openHostApp.failed", extra: "path=startflow cancelPending=1")
@@ -425,8 +514,10 @@ final class KeyboardFlowCoordinator {
ExtensionScreenWakeLock.release()
}
currentUtteranceId = nil
lastStoppedUtteranceId = nil
isFlowRecording = false
isPendingFlowStart = false
recordAfterHandoff = false
stopUtteranceCountdown()
stopFlowWatchdog()
state.level = 0
@@ -472,6 +563,7 @@ final class KeyboardFlowCoordinator {
)
FlowSessionBridge.clearResult()
lastConsumedUtteranceId = result.utteranceId
lastStoppedUtteranceId = nil
currentUtteranceId = nil
textInserter.handleFlowTranscript(
TranscriptionDelivery(text: text, polishWarning: result.warning)
@@ -483,6 +575,7 @@ final class KeyboardFlowCoordinator {
stopFlowWatchdog()
FlowSessionBridge.clearResult()
lastConsumedUtteranceId = result.utteranceId
lastStoppedUtteranceId = nil
currentUtteranceId = nil
let error = FlowTranscriptionError(
message: result.text ?? ExtL10n.string("keyboard.flow.resultTimeout"),
@@ -528,6 +621,7 @@ final class KeyboardFlowCoordinator {
ExtensionScreenWakeLock.release()
writeCommand(.abort)
currentUtteranceId = nil
lastStoppedUtteranceId = nil
stopFlowWatchdog()
state.level = 0
state.phase = .idle
@@ -547,10 +641,12 @@ final class KeyboardFlowCoordinator {
isAwaitingFlowResult = false
isFlowRecording = false
isPendingFlowStart = false
recordAfterHandoff = false
stopUtteranceCountdown()
ExtensionScreenWakeLock.release()
writeCommand(.abort)
currentUtteranceId = nil
lastStoppedUtteranceId = nil
stopFlowWatchdog()
state.level = 0
let message = ExtL10n.string("keyboard.flow.hostDisconnected")
@@ -585,9 +681,29 @@ final class KeyboardFlowCoordinator {
private func startFlowRecording() {
recomputeMicVoiceAvailability()
guard state.micVoiceAvailability.isReady else {
traceState("startFlowRecording.blocked", extra: "availability=\(String(describing: state.micVoiceAvailability))")
beginFlowStart()
let withinReadyGrace = lastHostReadyAt > 0
&& (Date().timeIntervalSince1970 - lastHostReadyAt) <= Self.hostReadyGrace
if !state.micVoiceAvailability.isReady {
let action = FlowHandoffPolicy.micPressAction(
availability: state.micVoiceAvailability,
sessionActive: FlowSessionBridge.isSessionActive(),
hostReachable: FlowSessionBridge.isHostReachable(),
hostStale: FlowSessionBridge.isHostStale(),
withinReadyGrace: withinReadyGrace
)
traceState(
"startFlowRecording.blocked",
extra: "availability=\(String(describing: state.micVoiceAvailability)) action=\(action)"
)
switch action {
case .waitForHostReady(let recordWhenReady):
recordWhenHostReady = recordWhenReady
startHostReadyWaitIfNeeded()
case .openHostColdStart:
beginFlowStart(recordAfterHandoff: true)
case .startRecording, .ignore:
break
}
return
}
isPendingFlowStart = false
@@ -596,11 +712,23 @@ final class KeyboardFlowCoordinator {
guard let sessionId = FlowSessionBridge.readySnapshot()?.sessionId else {
traceState("startFlowRecording.blocked", extra: "reason=missingSessionIdInReadySnapshot")
beginFlowStart()
// Snapshot lag with a live session wait; only cold-start if host is dead.
if FlowHandoffPolicy.shouldOpenHostColdStart(
sessionActive: FlowSessionBridge.isSessionActive(),
hostReachable: FlowSessionBridge.isHostReachable(),
hostStale: FlowSessionBridge.isHostStale(),
withinReadyGrace: withinReadyGrace
) {
beginFlowStart(recordAfterHandoff: true)
} else {
recordWhenHostReady = true
startHostReadyWaitIfNeeded()
}
return
}
activeSessionId = sessionId
currentUtteranceId = UUID()
lastStoppedUtteranceId = nil
writeCommand(.startRecording)
isFlowRecording = true
state.lastTranscript = ""
@@ -640,8 +768,12 @@ final class KeyboardFlowCoordinator {
private func cancelPendingFlowStart() {
isPendingFlowStart = false
recordAfterHandoff = false
recordWhenHostReady = false
flowStartDeadline = 0
coldStartDebouncer.reset()
stopFlowWatchdog()
stopHostReadyWait()
state.phase = .idle
state.lastTranscript = ""
recomputeMicVoiceAvailability()
@@ -660,6 +792,7 @@ final class KeyboardFlowCoordinator {
let now = Date().timeIntervalSince1970
if self.flowStartDeadline > 0, now > self.flowStartDeadline {
self.isPendingFlowStart = false
self.recordAfterHandoff = false
self.flowStartDeadline = 0
self.traceState("startWatchdog.timeout")
self.showManualOpenHint(path: "startflow")
@@ -671,13 +804,20 @@ final class KeyboardFlowCoordinator {
}
private func completeFlowStartHandoff() {
let shouldRecord = recordAfterHandoff
isPendingFlowStart = false
recordAfterHandoff = false
flowStartDeadline = 0
stopFlowWatchdog()
state.lastTranscript = ""
refreshSessionState()
startFlowRecording()
traceState("completeFlowStartHandoff.done")
if shouldRecord {
startFlowRecording()
traceState("completeFlowStartHandoff.done", extra: "record=1")
} else {
recomputeMicVoiceAvailability()
traceState("completeFlowStartHandoff.done", extra: "record=0 warmOnly")
}
}
private func startFlowLevelWatchdog() {
@@ -735,6 +875,7 @@ final class KeyboardFlowCoordinator {
)
FlowSessionBridge.clearResult()
self.lastConsumedUtteranceId = result.utteranceId
self.lastStoppedUtteranceId = nil
self.currentUtteranceId = nil
self.debug("resultWatchdog consumed delivery len=\(text.count)")
self.textInserter.handleFlowTranscript(
@@ -747,6 +888,7 @@ final class KeyboardFlowCoordinator {
self.stopFlowWatchdog()
FlowSessionBridge.clearResult()
self.lastConsumedUtteranceId = result.utteranceId
self.lastStoppedUtteranceId = nil
self.currentUtteranceId = nil
let error = FlowTranscriptionError(
message: result.text ?? ExtL10n.string("keyboard.flow.resultTimeout"),
@@ -784,6 +926,7 @@ final class KeyboardFlowCoordinator {
self.isAwaitingFlowResult = false
self.stopFlowWatchdog()
self.currentUtteranceId = nil
self.lastStoppedUtteranceId = nil
self.debug("resultWatchdog TIMEOUT after \(Int(resultTimeout))s — no result from host")
let msg = ExtL10n.string("keyboard.flow.resultTimeout")
self.state.phase = .error(.flowResultTimeout, message: msg)
+12 -11
View File
@@ -12,7 +12,7 @@
// (transcript preview)
//
// mic (centred) action cluster:
// [delete] [ space ] [return] mic + bottom row
// [delete] [ return ] [space] mic + bottom row
//
//
@@ -191,7 +191,7 @@ public struct KeyboardRootView: View {
// MARK: - Action cluster
/// Mic centred above a bottom row: delete · space · return (or swapped).
/// Mic centred above a bottom row: delete · smart return · space (or swapped).
/// The side cursor-drag pads are SwiftUI layout wrappers around UIKit
/// pan recognizers, avoiding SwiftUI gesture delivery issues in
/// keyboard extensions.
@@ -225,13 +225,13 @@ public struct KeyboardRootView: View {
HStack(spacing: KeyboardLayoutMetrics.bottomActionSpacing) {
if swapKeys {
bottomReturnButton(disabled: editingBlocked)
bottomSpaceButton(disabled: editingBlocked)
bottomReturnButton(disabled: editingBlocked)
bottomDeleteButton(disabled: editingBlocked)
} else {
bottomDeleteButton(disabled: editingBlocked)
bottomSpaceButton(disabled: editingBlocked)
bottomReturnButton(disabled: editingBlocked)
bottomSpaceButton(disabled: editingBlocked)
}
}
.opacity(dragging ? 0 : 1)
@@ -265,19 +265,20 @@ public struct KeyboardRootView: View {
RectangularToolbarButton(spaceStyle: true, label: "space", disabled: disabled) {
state.insertSpace()
}
.frame(height: KeyboardLayoutMetrics.bottomActionRowHeight)
}
private func bottomReturnButton(disabled: Bool) -> some View {
RectangularToolbarButton(systemName: "return", label: "newline", disabled: disabled) {
state.insertNewline()
}
.frame(
width: KeyboardLayoutMetrics.bottomActionFixedWidth,
height: KeyboardLayoutMetrics.bottomActionRowHeight
)
}
private func bottomReturnButton(disabled: Bool) -> some View {
let title = ExtL10n.string(state.returnKeyRole.titleKey)
return RectangularToolbarButton(title: title, label: title, disabled: disabled) {
state.insertNewline()
}
.frame(height: KeyboardLayoutMetrics.bottomActionRowHeight)
}
/// Option C: block typing keys during the full voice-input pipeline.
private var voiceInputBlocksEditing: Bool {
switch state.phase {
@@ -10,6 +10,7 @@ import OSGKeyboardShared
private enum ToolbarButtonMetrics {
static let iconSize: CGFloat = 14
static let titleSize: CGFloat = 16
static let cornerRadius: CGFloat = 12
static let spaceBarCapsuleWidth: CGFloat = 31
static let pressScale: CGFloat = 0.94
@@ -138,6 +139,7 @@ struct RectangularToolbarButton: View {
let systemName: String?
let spaceStyle: Bool
let title: String?
let label: String
let disabled: Bool
let action: () -> Void
@@ -145,14 +147,25 @@ struct RectangularToolbarButton: View {
init(systemName: String, label: String, disabled: Bool = false, action: @escaping () -> Void) {
self.systemName = systemName
self.spaceStyle = false
self.title = nil
self.label = label
self.disabled = disabled
self.action = action
}
init(title: String, label: String, disabled: Bool = false, action: @escaping () -> Void) {
self.systemName = nil
self.spaceStyle = false
self.label = label
self.disabled = disabled
self.action = action
self.title = title
}
init(spaceStyle: Bool, label: String, disabled: Bool = false, action: @escaping () -> Void) {
self.systemName = nil
self.spaceStyle = spaceStyle
self.title = nil
self.label = label
self.disabled = disabled
self.action = action
@@ -170,6 +183,10 @@ struct RectangularToolbarButton: View {
Image(systemName: systemName)
.font(.system(size: ToolbarButtonMetrics.iconSize, weight: .semibold))
.foregroundStyle(palette.textPrimary)
} else if let title {
Text(title)
.font(.system(size: ToolbarButtonMetrics.titleSize, weight: .semibold))
.foregroundStyle(palette.textPrimary)
}
}
.contentShape(Rectangle())
+7 -6
View File
@@ -25,6 +25,7 @@
"common.delete" = "Delete";
"common.space" = "Space";
"common.newline" = "Return";
"common.send" = "Send";
/* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "On-device transcription";
@@ -69,13 +70,13 @@
"settings.reset.title" = "Reset all settings?";
"settings.reset.message" = "API key, model, and base URL will be cleared.";
"settings.reset.confirm" = "Reset all settings";
"settings.engine.title" = "Recognition method";
"settings.engine.subtitle" = "Pick the recognition engine. Local engine does transcription only, no API key needed.";
"settings.engine.local.title" = "On-device recognition";
"settings.engine.title" = "Speech transcription method";
"settings.engine.subtitle" = "Local: built-in system transcription. Cloud: ASR transcription with optional LLM polish.";
"settings.engine.local.title" = "On-device transcription";
"settings.engine.local.ios26" = "Always on-device, no network.";
"settings.engine.local.legacy" = "On-device ASR. Transcription only, no polish.";
"settings.engine.cloud.title" = "Cloud recognition & polish";
"settings.engine.cloud.subtitle" = "ASR + LLM polish. API key required.";
"settings.engine.local.legacy" = "Transcribe with the built-in system speech service";
"settings.engine.cloud.title" = "Cloud transcription";
"settings.engine.cloud.subtitle" = "Cloud ASR transcription, with optional LLM polish";
"settings.provider.title" = "Provider";
"settings.provider.subtitle" = "Pick the LLM that polishes your dictation.";
"settings.api.title" = "API";
@@ -24,7 +24,8 @@
"common.cancel" = "取消";
"common.delete" = "删除";
"common.space" = "空格";
"common.newline" = "换行";
"common.newline" = "回车";
"common.send" = "发送";
/* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "支持本地转写";
@@ -69,13 +70,13 @@
"settings.reset.title" = "重置所有设置?";
"settings.reset.message" = "API key、model 和 base URL 都会被清空。";
"settings.reset.confirm" = "重置所有设置";
"settings.engine.title" = "识别方式";
"settings.engine.subtitle" = "选择识别方式。本地引擎无需 API Key,仅做语音转录。";
"settings.engine.local.title" = "本地识别";
"settings.engine.title" = "语音转写方式";
"settings.engine.subtitle" = "本地:系统内置转写;云端:ASR 转写,可用 LLM 润色。";
"settings.engine.local.title" = "本地转写";
"settings.engine.local.ios26" = "始终端侧,无需联网。";
"settings.engine.local.legacy" = "端侧 ASR,仅转录,无润色。";
"settings.engine.cloud.title" = "云端识别与润色";
"settings.engine.cloud.subtitle" = "ASR 转录 + LLM 润色,需要 API Key。";
"settings.engine.local.legacy" = "通过系统内置服务语音转写";
"settings.engine.cloud.title" = "云端转写";
"settings.engine.cloud.subtitle" = "通过云端 ASR 转写,可用 LLM 润色";
"settings.provider.title" = "云端引擎";
"settings.provider.subtitle" = "选择 LLM 提供商。";
"settings.api.title" = "接口";