feat(macos): add macOS menu-bar app and harden cross-device iCloud sync

Introduce a standalone macOS menu-bar app (OSGKeyboardMac) that reuses the
platform-agnostic OSGKeyboardShared core: record -> cloud/local ASR -> polish
-> insert. Local mode uses Qwen3-ASR via mlx-swift-asr (macOS 15+, Apple
Silicon); iOS targets stay zero-SPM.

Harden iCloud sync for multi-device correctness:
- Per-field settings merge (appSettings.v2) so concurrent edits no longer
  clobber each other's unrelated fields.
- Per-device usage statistics (G-Counter) that sum instead of max().
- Tombstoned dictionary/history merge so deletes propagate and entries can't
  resurrect.
- API keys replicate via iCloud Keychain, never iCloud KVS JSON; pulling a
  legacy blob without key fields no longer wipes local Keychain entries.
- Add a low-risk "Sync Now" action in Settings.

Fix Flow keyboard mic state: stay orange until the host publishes a real ready
contract, share a single MicVoiceAvailability gate, and self-heal stale
cross-process heartbeat jitter instead of getting stuck.

Extract shared storage (SpeechHistoryStore/UsageStatisticsStore,
ConfigurationStore) into OSGKeyboardShared and add tests for the new
sync/merge logic.
This commit is contained in:
Rocky
2026-07-08 18:13:56 +08:00
parent 128aab1b02
commit c2f07bd8d2
99 changed files with 6735 additions and 740 deletions
@@ -313,6 +313,10 @@ public final class KeyboardViewController: UIInputViewController {
switch self.state.phase {
case .error:
self.state.phase = .idle
// Re-derive mic availability right away so a now-ready host
// turns the mic green immediately instead of lingering orange
// until the next monitor tick.
self.flowCoordinator.refreshSessionState()
default:
break
}
@@ -18,6 +18,7 @@ final class KeyboardConfigSync {
private var flowSessionDarwinObserver: FlowSessionDarwinObserver?
private var transcriptionDarwinObserver: FlowSessionDarwinObserver?
private var hostReadyDarwinObserver: FlowSessionDarwinObserver?
private var configDarwinObserver: FlowSessionDarwinObserver?
init(
@@ -39,6 +40,11 @@ final class KeyboardConfigSync {
) { [weak self] in
self?.onFlowSessionChanged()
}
hostReadyDarwinObserver = FlowSessionDarwinObserver(
notificationName: FlowSessionDarwin.hostReadyNotificationName
) { [weak self] in
self?.onFlowSessionChanged()
}
configDarwinObserver = FlowSessionDarwinObserver(
notificationName: AppGroupConfigDarwin.notificationName
) { [weak self] in
@@ -32,12 +32,16 @@ final class KeyboardFlowCoordinator {
private var isFlowRecording = false
private var flowWatchdogTask: Task<Void, Never>?
private var utteranceTimerTask: Task<Void, Never>?
private var hostReadyWaitTask: Task<Void, Never>?
private var utteranceStartedAt: TimeInterval = 0
private var wasFlowSessionActive = false
private var wasSessionActive = false
/// Last wall-clock time the host published a fresh ready contract. Used to
/// smooth over transient cross-process heartbeat read jitter so a single
/// stale sample never flashes the mic orange while the session is healthy.
private var lastHostReadyAt: TimeInterval = 0
private static let hostReadyGrace: TimeInterval = 4
private var flowSessionMonitorTask: Task<Void, Never>?
private var isAwaitingFlowResult = false
private var lastFlowAutoStartAttempt: TimeInterval = 0
private static let flowAutoStartCooldown: TimeInterval = 20
init(
state: KeyboardState,
@@ -82,9 +86,11 @@ final class KeyboardFlowCoordinator {
func stopSessionMonitor() {
flowSessionMonitorTask?.cancel()
flowSessionMonitorTask = nil
stopHostReadyWait()
}
func refreshSessionState() {
FlowSessionBridge.reloadFromDisk()
refreshConfigFromAppGroup()
refreshFlowPartialIfNeeded()
consumePendingFlowDeliveryIfNeeded()
@@ -95,10 +101,26 @@ final class KeyboardFlowCoordinator {
debug("cleared zombie Flow session from App Group")
}
let reachable = FlowSessionBridge.isHostReachable()
state.flowSessionActive = reachable
// A stale "session ended" hint may linger from an earlier drop. If the
// host is provably ready again, recover to idle now so the mic can go
// green immediately instead of waiting out the auto-clear timer.
if case .error(.flowSessionExpired, _) = state.phase,
FlowSessionBridge.isHostReady() {
state.phase = .idle
state.lastTranscript = ""
}
if wasFlowSessionActive && !reachable && !isFlowRecording && !isPendingFlowStart {
recomputeMicVoiceAvailability()
startHostReadyWaitIfNeeded()
// Only surface "session ended" when the session contract *genuinely*
// dropped (expired / cleared). A transient host-ready flap engine
// hiccup or a stale cross-process read while the session is still
// valid must never nuke a healthy ready state into a sticky error,
// otherwise the error phase forces the mic orange and defeats the
// ready-wait poll until the auto-clear fires.
let sessionActive = FlowSessionBridge.isSessionActive()
if wasSessionActive && !sessionActive && !isFlowRecording && !isPendingFlowStart {
switch state.phase {
case .recording, .processing:
break
@@ -106,9 +128,65 @@ final class KeyboardFlowCoordinator {
showFlowSessionExpiredHint()
}
}
wasFlowSessionActive = reachable
wasSessionActive = sessionActive
}
maybeAutoStartFlowSession()
private func recomputeMicVoiceAvailability() {
FlowSessionBridge.reloadFromDisk()
let hostReady = FlowSessionBridge.isHostReady()
let now = Date().timeIntervalSince1970
if hostReady { lastHostReadyAt = now }
// Grace window: the host was ready very recently, so treat a momentary
// stale heartbeat read as "still warming" rather than an outright
// failure. `isSessionActive` is heartbeat-independent, so it stays true
// across cross-process read jitter and anchors this smoothing.
let withinReadyGrace = lastHostReadyAt > 0
&& (now - lastHostReadyAt) <= Self.hostReadyGrace
let hostWarming = !hostReady
&& FlowSessionBridge.isSessionActive()
&& (FlowSessionBridge.isHostReachable() || isPendingFlowStart || withinReadyGrace)
state.flowSessionActive = hostReady
state.micVoiceAvailability = MicVoiceAvailabilityResolver.resolve(
phase: state.phase,
micDisabled: state.micDisabled,
hasFullAccess: hasFullAccess(),
appGroupAvailable: AppGroup.isAvailable,
hostReady: hostReady,
isPreparingSession: isPendingFlowStart || hostWarming
)
}
/// Session is live but the ready contract has not landed yet poll
/// quickly instead of sticking on "session inactive" orange.
private func startHostReadyWaitIfNeeded() {
guard !isPendingFlowStart else { return }
guard FlowSessionBridge.isSessionActive() else {
stopHostReadyWait()
return
}
guard !FlowSessionBridge.isHostReady() else {
stopHostReadyWait()
return
}
guard hostReadyWaitTask == nil else { return }
hostReadyWaitTask = Task { @MainActor [weak self] in
defer { self?.hostReadyWaitTask = nil }
for _ in 0..<20 {
guard let self, !Task.isCancelled else { return }
FlowSessionBridge.reloadFromDisk()
self.recomputeMicVoiceAvailability()
if self.state.micVoiceAvailability.isReady {
return
}
try? await Task.sleep(nanoseconds: 150_000_000)
}
}
}
private func stopHostReadyWait() {
hostReadyWaitTask?.cancel()
hostReadyWaitTask = nil
}
func toggleRecording() {
@@ -129,32 +207,33 @@ final class KeyboardFlowCoordinator {
default:
return
}
guard !state.micDisabled else { return }
guard hasFullAccess() else {
guard !isPendingFlowStart else { return }
recomputeMicVoiceAvailability()
switch state.micVoiceAvailability {
case .ready:
detectAndStoreAppContext()
startFlowRecording()
case .unavailable(.missingAPIKey):
return
case .unavailable(.noFullAccess):
let msg = ExtL10n.string("keyboard.error.fullAccessRequired")
state.phase = .error(.fullAccessRequired, message: msg)
scheduleAutoClearError()
return
}
guard AppGroup.isAvailable else {
recomputeMicVoiceAvailability()
case .unavailable(.appGroupUnavailable):
let msg = ExtL10n.string("keyboard.error.appGroupCommunication")
state.phase = .error(.appGroupUnavailable, message: msg)
scheduleAutoClearError()
recomputeMicVoiceAvailability()
case .unavailable(.preparingSession):
return
}
detectAndStoreAppContext()
let reachable = FlowSessionBridge.isHostReachable()
debug(
"pressBegan hostReachable=\(reachable) " +
"staleness=\(FlowSessionBridge.heartbeatStaleness().map { String(format: "%.1f", $0) } ?? "nil") " +
"container=\(AppGroup.containerPathForDiagnostics)"
)
if reachable {
startFlowRecording()
} else {
case .unavailable(.hostNotReady):
detectAndStoreAppContext()
beginFlowStart()
case .recording, .processing:
return
}
}
@@ -172,6 +251,7 @@ final class KeyboardFlowCoordinator {
debug("pressEnded wrote .stopped (readback=\(FlowSessionBridge.recordingState().rawValue))")
state.phase = .processing
state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
recomputeMicVoiceAvailability()
startFlowResultWatchdog()
}
@@ -181,7 +261,7 @@ final class KeyboardFlowCoordinator {
isFlowRecording = false
flowStartDeadline = Date().timeIntervalSince1970 + FlowWatchdog.startTimeout
state.lastTranscript = ExtL10n.string("keyboard.flow.startingSession")
state.phase = .processing
recomputeMicVoiceAvailability()
openHostApp("startflow")
startFlowStartWatchdog()
debug("beginFlowStart")
@@ -199,6 +279,7 @@ final class KeyboardFlowCoordinator {
flowStartDeadline = 0
stopFlowWatchdog()
showManualOpenHint(path: "startflow")
recomputeMicVoiceAvailability()
return
}
@@ -217,6 +298,7 @@ final class KeyboardFlowCoordinator {
stopUtteranceCountdown()
stopFlowWatchdog()
state.level = 0
recomputeMicVoiceAvailability()
}
}
@@ -238,11 +320,12 @@ final class KeyboardFlowCoordinator {
message: error.message
)
scheduleAutoClearError()
recomputeMicVoiceAvailability()
return
}
}
if isPendingFlowStart, FlowSessionBridge.isHostReachable() {
if isPendingFlowStart, FlowSessionBridge.isHostReady() {
completeFlowStartHandoff()
}
}
@@ -261,6 +344,7 @@ final class KeyboardFlowCoordinator {
state.level = 0
state.phase = .idle
state.lastTranscript = ""
recomputeMicVoiceAvailability()
debug("aborted recording — host heartbeat zombie")
return
}
@@ -270,20 +354,6 @@ final class KeyboardFlowCoordinator {
}
}
/// Restores the pre-ABCD behaviour: when the keyboard appears and the host
/// is not reachable, automatically jump to the main app to start Flow.
private func maybeAutoStartFlowSession() {
guard !FlowSessionBridge.isHostReachable() else { return }
guard !isPendingFlowStart, !isFlowRecording, !isAwaitingFlowResult else { return }
guard hasFullAccess(), AppGroup.isAvailable else { return }
guard case .idle = state.phase else { return }
let now = Date().timeIntervalSince1970
guard now - lastFlowAutoStartAttempt >= Self.flowAutoStartCooldown else { return }
lastFlowAutoStartAttempt = now
beginFlowStart()
}
private func failHostDisconnected() {
isAwaitingFlowResult = false
isFlowRecording = false
@@ -296,6 +366,7 @@ final class KeyboardFlowCoordinator {
let message = ExtL10n.string("keyboard.flow.hostDisconnected")
state.phase = .error(.flowSessionExpired, message: message)
scheduleAutoClearError()
recomputeMicVoiceAvailability()
debug("host disconnected while awaiting Flow result")
}
@@ -303,6 +374,7 @@ final class KeyboardFlowCoordinator {
let message = ExtL10n.string("keyboard.flow.sessionExpired")
state.phase = .error(.flowSessionExpired, message: message)
scheduleAutoClearError()
recomputeMicVoiceAvailability()
}
private func showManualOpenHint(path: String) {
@@ -318,10 +390,12 @@ final class KeyboardFlowCoordinator {
}
state.phase = .error(.manualOpenRequired, message: msg)
scheduleAutoClearError()
recomputeMicVoiceAvailability()
}
private func startFlowRecording() {
guard FlowSessionBridge.isHostReachable() else {
recomputeMicVoiceAvailability()
guard state.micVoiceAvailability.isReady else {
beginFlowStart()
return
}
@@ -334,6 +408,7 @@ final class KeyboardFlowCoordinator {
isFlowRecording = true
state.lastTranscript = ""
state.phase = .recording
recomputeMicVoiceAvailability()
if let view = wakeLockView() {
ExtensionScreenWakeLock.acquire(from: view)
}
@@ -374,13 +449,15 @@ final class KeyboardFlowCoordinator {
stopFlowWatchdog()
state.phase = .idle
state.lastTranscript = ""
recomputeMicVoiceAvailability()
}
private func startFlowStartWatchdog() {
stopFlowWatchdog()
flowWatchdogTask = Task { @MainActor [weak self] in
while let self, !Task.isCancelled, self.isPendingFlowStart {
if FlowSessionBridge.isHostReachable() {
self.recomputeMicVoiceAvailability()
if FlowSessionBridge.isHostReady() {
self.completeFlowStartHandoff()
return
}
@@ -456,6 +533,7 @@ final class KeyboardFlowCoordinator {
message: error.message
)
self.scheduleAutoClearError()
self.recomputeMicVoiceAvailability()
return
}
self.refreshFlowPartialIfNeeded()
@@ -479,6 +557,7 @@ final class KeyboardFlowCoordinator {
let msg = ExtL10n.string("keyboard.flow.resultTimeout")
self.state.phase = .error(.flowResultTimeout, message: msg)
self.scheduleAutoClearError()
self.recomputeMicVoiceAvailability()
return
}
try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs)
+81 -67
View File
@@ -138,8 +138,7 @@ public struct KeyboardRootView: View {
TranscriptLine(
phase: state.phase,
transcript: state.lastTranscript,
flowSessionActive: state.flowSessionActive,
micDisabled: state.micDisabled,
micVoiceAvailability: state.micVoiceAvailability,
micDisabledHint: state.micDisabledHint,
cursorDragHintActive: state.cursorDragActive,
openSettings: state.openSettings
@@ -192,7 +191,6 @@ public struct KeyboardRootView: View {
private var micActionRow: some View {
let editingBlocked = voiceInputBlocksEditing
let swapKeys = state.handednessPreference.swapsActionKeys
let micDisabled = state.micDisabled
let cursorPadsEnabled = state.cursorDragNavigationEnabled && !editingBlocked
// Dragging hides the mic + bottom keys (kept in the layout via
@@ -208,7 +206,7 @@ public struct KeyboardRootView: View {
phase: buttonPhase,
level: state.level,
remainingSeconds: state.phase == .recording ? state.utteranceRemainingSeconds : nil,
isEnabled: !micDisabled,
isEnabled: !state.micDisabled,
onToggle: state.tapMic
)
.frame(width: KeyboardLayoutMetrics.micSize, height: KeyboardLayoutMetrics.micSize)
@@ -284,13 +282,17 @@ public struct KeyboardRootView: View {
}
private var buttonPhase: RecordButton.Phase {
switch state.phase {
case .idle: return .idle
case .requestingPermissions: return .idle
case .recording: return .recording
case .processing: return .processing
case .error: return .error
case .denied: return .error
if case .error = state.phase { return .error }
if case .denied = state.phase { return .error }
switch state.micVoiceAvailability {
case .ready:
return .idleReady
case .unavailable:
return .idleUnavailable
case .recording:
return .recording
case .processing:
return .processing
}
}
}
@@ -330,8 +332,7 @@ private struct TranscriptLine: View {
let phase: KeyboardViewController.State.Phase
let transcript: String
let flowSessionActive: Bool
let micDisabled: Bool
let micVoiceAvailability: MicVoiceAvailability
let micDisabledHint: String
let cursorDragHintActive: Bool
let openSettings: () -> Void
@@ -353,66 +354,79 @@ private struct TranscriptLine: View {
private var phaseContent: some View {
switch phase {
case .idle:
if micDisabled {
Text(micDisabledHint)
.font(TypeStyle.caption)
.foregroundStyle(palette.warning)
.lineLimit(1)
.truncationMode(.tail)
} else if flowSessionActive {
ExtL10n.text("keyboard.placeholder.idle")
.font(TypeStyle.caption)
.foregroundStyle(palette.textTertiary)
} else {
ExtL10n.text("keyboard.flow.sessionInactive")
.font(TypeStyle.caption)
.foregroundStyle(palette.textTertiary)
}
case .requestingPermissions:
HStack(spacing: 6) {
ProgressView().controlSize(.mini).tint(palette.textSecondary)
ExtL10n.text("keyboard.placeholder.preparing")
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
}
case .recording:
Text(transcript.isEmpty ? " " : transcript)
.font(TypeStyle.caption)
.foregroundStyle(palette.textPrimary)
.lineLimit(1)
.truncationMode(.head)
.frame(maxWidth: .infinity)
case .processing:
Text(transcript.isEmpty ? ExtL10n.string("keyboard.placeholder.processing") : transcript)
idleHint
case .requestingPermissions:
HStack(spacing: 6) {
ProgressView().controlSize(.mini).tint(palette.textSecondary)
ExtL10n.text("keyboard.placeholder.preparing")
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
.lineLimit(1)
.truncationMode(.tail)
case .error(_, let msg):
Text(msg ?? "")
.font(TypeStyle.caption)
.foregroundStyle(palette.warning)
.lineLimit(1)
.truncationMode(.tail)
case .denied(let reason):
Button(action: openSettings) {
HStack(spacing: 4) {
Text(deniedMessage(for: reason))
.lineLimit(1)
.truncationMode(.tail)
Image(systemName: "chevron.right")
.font(.system(size: 10, weight: .semibold))
}
.font(TypeStyle.caption)
.foregroundStyle(palette.warning)
.frame(maxWidth: .infinity)
.contentShape(Rectangle())
}
case .recording:
Text(transcript.isEmpty ? " " : transcript)
.font(TypeStyle.caption)
.foregroundStyle(palette.textPrimary)
.lineLimit(1)
.truncationMode(.head)
.frame(maxWidth: .infinity)
case .processing:
Text(transcript.isEmpty ? ExtL10n.string("keyboard.placeholder.processing") : transcript)
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
.lineLimit(1)
.truncationMode(.tail)
case .error(_, let msg):
Text(msg ?? "")
.font(TypeStyle.caption)
.foregroundStyle(palette.warning)
.lineLimit(1)
.truncationMode(.tail)
case .denied(let reason):
Button(action: openSettings) {
HStack(spacing: 4) {
Text(deniedMessage(for: reason))
.lineLimit(1)
.truncationMode(.tail)
Image(systemName: "chevron.right")
.font(.system(size: 10, weight: .semibold))
}
.buttonStyle(.plain)
.accessibilityHint(ExtL10n.text("keyboard.deniedHint"))
.font(TypeStyle.caption)
.foregroundStyle(palette.warning)
.frame(maxWidth: .infinity)
.contentShape(Rectangle())
}
.buttonStyle(.plain)
.accessibilityHint(ExtL10n.text("keyboard.deniedHint"))
}
}
@ViewBuilder
private var idleHint: some View {
let isWarning = micVoiceAvailability.isUnavailable
Group {
switch micVoiceAvailability {
case .ready:
ExtL10n.text("keyboard.placeholder.idle")
case .unavailable(.missingAPIKey):
Text(micDisabledHint)
case .unavailable(.hostNotReady):
ExtL10n.text("keyboard.flow.sessionInactive")
case .unavailable(.preparingSession):
ExtL10n.text("keyboard.flow.startingSession")
case .unavailable(.noFullAccess):
ExtL10n.text("keyboard.error.fullAccessRequired")
case .unavailable(.appGroupUnavailable):
ExtL10n.text("keyboard.error.appGroupCommunication")
case .recording, .processing:
EmptyView()
}
}
.font(TypeStyle.caption)
.foregroundStyle(isWarning ? palette.warning : palette.textTertiary)
.lineLimit(1)
.truncationMode(.tail)
}
private func deniedMessage(for reason: KeyboardViewController.State.Phase.Reason) -> String {
switch reason {
case .mic: return ExtL10n.string("keyboard.denied.mic")