diff --git a/.gitignore b/.gitignore index 0f0ce23..4505446 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,6 @@ Signing.local.xcconfig # Generated by XcodeGen *.xcodeproj/ + +# Local DerivedData (when using -derivedDataPath in repo) +.derivedData/ diff --git a/OSGKeyboard/Assets.xcassets/AppIcon.appiconset/Group 24.png b/OSGKeyboard/Assets.xcassets/AppIcon.appiconset/Group 24.png index d80ad7f..7dc4e6a 100644 Binary files a/OSGKeyboard/Assets.xcassets/AppIcon.appiconset/Group 24.png and b/OSGKeyboard/Assets.xcassets/AppIcon.appiconset/Group 24.png differ diff --git a/OSGKeyboard/Info.plist b/OSGKeyboard/Info.plist index c0b8d78..0583e58 100644 --- a/OSGKeyboard/Info.plist +++ b/OSGKeyboard/Info.plist @@ -42,14 +42,18 @@ NSMicrophoneUsageDescription - OSGKeyboard needs microphone access to transcribe your voice into text. + OSGKeyboard uses the microphone for voice dictation and keeps a background audio session active while a voice session is running. NSSpeechRecognitionUsageDescription - OSGKeyboard uses on-device speech recognition to transcribe your dictation. Audio never leaves your device. + OSGKeyboard uses on-device speech recognition to transcribe your voice. Audio is processed on your device and is not uploaded for transcription. UIApplicationSceneManifest UIApplicationSupportsMultipleScenes + UIBackgroundModes + + audio + UILaunchScreen UIColorName diff --git a/OSGKeyboard/OSGKeyboard.entitlements b/OSGKeyboard/OSGKeyboard.entitlements index f8f12d1..fb3b8f1 100644 --- a/OSGKeyboard/OSGKeyboard.entitlements +++ b/OSGKeyboard/OSGKeyboard.entitlements @@ -8,9 +8,9 @@ com.apple.security.device.audio-input - com.apple.security.keychain-access-groups + keychain-access-groups - com.osgkeyboard.shared + $(AppIdentifierPrefix)com.osgkeyboard.shared diff --git a/OSGKeyboard/OSGKeyboardApp.swift b/OSGKeyboard/OSGKeyboardApp.swift index e5ae260..df8c0a3 100644 --- a/OSGKeyboard/OSGKeyboardApp.swift +++ b/OSGKeyboard/OSGKeyboardApp.swift @@ -6,14 +6,19 @@ import OSGKeyboardShared @main struct OSGKeyboardApp: App { + @Environment(\.scenePhase) private var scenePhase + @StateObject private var config = ProviderConfig.shared + @StateObject private var dictationCoordinator = DictationSessionCoordinator() + @StateObject private var flowManager = FlowSessionManager() var body: some Scene { WindowGroup { ThemedRoot { if AppGroup.isAvailable { - if config.isConfigured { + if config.hasCompletedOnboarding { HomeView() + .onAppear { flowManager.autoStartIfNeeded() } } else { OnboardingView(config: config) } @@ -21,6 +26,32 @@ struct OSGKeyboardApp: App { AppGroupErrorView() } } + .environmentObject(flowManager) + .onOpenURL { url in + guard url.scheme == "osgkeyboard" else { return } + switch url.host { + case "dictate": + dictationCoordinator.present() + case "startflow": + flowManager.startSession() + default: + break + } + } + .fullScreenCover(isPresented: $dictationCoordinator.isPresenting) { + DictationCaptureView( + config: config, + coordinator: dictationCoordinator + ) + } + .onChange(of: scenePhase) { _, phase in + guard phase == .active, AppGroup.isAvailable, config.hasCompletedOnboarding else { return } + if flowManager.isActive { + flowManager.extendSession() + } else { + flowManager.autoStartIfNeeded() + } + } } } -} \ No newline at end of file +} diff --git a/OSGKeyboard/Services/AppPermissions.swift b/OSGKeyboard/Services/AppPermissions.swift new file mode 100644 index 0000000..5e73946 --- /dev/null +++ b/OSGKeyboard/Services/AppPermissions.swift @@ -0,0 +1,79 @@ +// AppPermissions.swift +// OSGKeyboard · Main App +// +// Central permission status for onboarding and Flow session startup. + +import AVFoundation +import Speech +import UIKit + +enum AppPermissions { + + enum MicStatus: Equatable { + case undetermined + case granted + case denied + } + + enum SpeechStatus: Equatable { + case undetermined + case granted + case denied + case restricted + } + + static var micStatus: MicStatus { + switch AVAudioApplication.shared.recordPermission { + case .granted: return .granted + case .denied: return .denied + case .undetermined: return .undetermined + @unknown default: return .denied + } + } + + static var speechStatus: SpeechStatus { + switch SFSpeechRecognizer.authorizationStatus() { + case .authorized: return .granted + case .denied: return .denied + case .restricted: return .restricted + case .notDetermined: return .undetermined + @unknown default: return .denied + } + } + + /// Both permissions required for Flow voice sessions. + static var flowRequirementsMet: Bool { + micStatus == .granted && speechStatus == .granted + } + + /// Show guided permission pages when any Flow permission is not granted. + static var needsPermissionGuidance: Bool { + micStatus != .granted || speechStatus != .granted + } + + static func requestMicrophone() async -> Bool { + switch micStatus { + case .granted: return true + case .denied: return false + case .undetermined: return await AVAudioApplication.requestRecordPermission() + } + } + + static func requestSpeechRecognition() async -> Bool { + switch speechStatus { + case .granted: return true + case .denied, .restricted: return false + case .undetermined: + return await withCheckedContinuation { (cont: CheckedContinuation) in + SFSpeechRecognizer.requestAuthorization { status in + cont.resume(returning: status == .authorized) + } + } + } + } + + static func openSystemSettings() { + guard let url = URL(string: UIApplication.openSettingsURLString) else { return } + UIApplication.shared.open(url) + } +} diff --git a/OSGKeyboard/Services/FlowSessionManager.swift b/OSGKeyboard/Services/FlowSessionManager.swift new file mode 100644 index 0000000..9dabd3b --- /dev/null +++ b/OSGKeyboard/Services/FlowSessionManager.swift @@ -0,0 +1,396 @@ +// FlowSessionManager.swift +// OSGKeyboard · Main App +// +// Session Owner for TypeWhisper-style Flow dictation: continuous +// `.playAndRecord` capture for the whole session, utterance gating for +// ASR, optional LLM polish, and App Group result delivery. + +import Foundation +import AVFoundation +import Speech +import OSGKeyboardShared + +@MainActor +final class FlowSessionManager: ObservableObject { + @Published private(set) var isActive = false + @Published private(set) var isStarting = false + @Published private(set) var sessionExpiresAt: Date? + /// Non-nil when continuous capture failed or permissions are missing. + @Published private(set) var sessionWarning: String? + + private let capture = FlowContinuousCapture() + private let asr: ASRService = ASRServiceFactory.make() + private let polisher = PolishingService() + private let store = AppGroupStore() + + private var pollingTask: Task? + private var heartbeatTask: Task? + private var expiryTask: Task? + private var levelTask: Task? + private var startTask: Task? + private var isUtteranceRecording = false + private var finalizeTask: Task? + private var asrTask: Task? + private var currentPartial = "" + private var lastFinal = "" + + init() { + Task { @MainActor [weak self] in + await self?.bootstrapFromStorageIfNeeded() + } + } + + // MARK: - Public + + /// Starts a Flow session: permissions → continuous capture → App Group active. + func startSession(duration: TimeInterval = FlowSessionKeys.defaultSessionDuration) { + guard AppGroup.isAvailable else { + debug("cannot start flow session: App Group unavailable") + return + } + + if isActive { + extendSession(duration: duration) + return + } + + startTask?.cancel() + startTask = Task { @MainActor [weak self] in + await self?.startSessionAsync(duration: duration) + } + } + + /// Called on launch / foreground when onboarding is complete. + func autoStartIfNeeded() { + guard AppGroup.isAvailable else { return } + guard !isActive, !isStarting else { return } + + guard AppPermissions.flowRequirementsMet else { + sessionWarning = permissionWarningMessage() + return + } + + let markedActive = AppGroup.defaults.bool(forKey: FlowSessionKeys.flowSessionActive) + if markedActive, FlowSessionBridge.remainingSessionDuration() != nil { + Task { await bootstrapFromStorageIfNeeded() } + return + } + + startSession() + } + + /// Reattach capture when the host app was killed but the session has not expired. + func bootstrapFromStorageIfNeeded() async { + guard AppGroup.isAvailable, !isActive else { return } + + let markedActive = AppGroup.defaults.bool(forKey: FlowSessionKeys.flowSessionActive) + guard markedActive, let remaining = FlowSessionBridge.remainingSessionDuration(), remaining > 0 else { + if markedActive { + FlowSessionBridge.markSessionInactive() + } + return + } + + guard AppPermissions.flowRequirementsMet else { + sessionWarning = permissionWarningMessage() + return + } + + isStarting = true + sessionWarning = nil + defer { isStarting = false } + + do { + try capture.start() + } catch { + let message = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + sessionWarning = message + FlowSessionBridge.markSessionInactive() + debug("bootstrap capture failed: \(message)") + return + } + + FlowSessionBridge.writeHeartbeat() + isActive = true + if let expires = FlowSessionBridge.sessionExpiresAt() { + sessionExpiresAt = Date(timeIntervalSince1970: expires) + } + + startHeartbeat() + startPolling() + startLevelPublishing() + scheduleExpiry(after: remaining) + + debug("Flow session restored (\(Int(remaining))s remaining)") + } + + func endSession() { + guard isActive else { return } + debug("Flow session ended") + + startTask?.cancel() + startTask = nil + pollingTask?.cancel() + pollingTask = nil + heartbeatTask?.cancel() + heartbeatTask = nil + expiryTask?.cancel() + expiryTask = nil + levelTask?.cancel() + levelTask = nil + finalizeTask?.cancel() + finalizeTask = nil + asrTask?.cancel() + asrTask = nil + + if isUtteranceRecording { + capture.cancelUtterance() + asr.cancel() + isUtteranceRecording = false + } + + capture.stop() + FlowSessionBridge.markSessionInactive() + isActive = false + sessionExpiresAt = nil + sessionWarning = nil + currentPartial = "" + lastFinal = "" + } + + func extendSession(duration: TimeInterval = FlowSessionKeys.defaultSessionDuration) { + FlowSessionBridge.extendSession(by: duration) + sessionExpiresAt = Date().addingTimeInterval(duration) + scheduleExpiry(after: duration) + } + + // MARK: - Session start + + private func startSessionAsync(duration: TimeInterval) async { + isStarting = true + sessionWarning = nil + defer { isStarting = false } + + guard AppPermissions.flowRequirementsMet else { + sessionWarning = permissionWarningMessage() + return + } + + do { + try capture.start() + } catch { + let message = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + sessionWarning = message + debug("continuous capture failed: \(message)") + return + } + + FlowSessionBridge.markSessionActive(duration: duration) + isActive = true + sessionExpiresAt = Date().addingTimeInterval(duration) + + startHeartbeat() + startPolling() + startLevelPublishing() + scheduleExpiry(after: duration) + + debug("Flow session started (\(Int(duration))s), continuous capture running") + } + + private func permissionWarningMessage() -> String { + if AppPermissions.micStatus != .granted { + return NSLocalizedString("flow.error.micRequired", comment: "") + } + return NSLocalizedString("flow.error.speechRequired", comment: "") + } + + // MARK: - Polling + + private func startPolling() { + pollingTask?.cancel() + pollingTask = Task { @MainActor [weak self] in + while !Task.isCancelled { + self?.handleKeyboardSignal() + try? await Task.sleep(nanoseconds: 100_000_000) + } + } + } + + private func handleKeyboardSignal() { + switch FlowSessionBridge.recordingState() { + case .recording: + guard !isUtteranceRecording else { return } + beginUtterance() + case .stopped: + guard isUtteranceRecording else { return } + endUtterance() + case .aborted: + abortUtterance() + case .idle, .processing: + break + } + } + + private func beginUtterance() { + guard capture.running else { + failUtterance(message: NSLocalizedString("flow.error.audioUnavailable", comment: "")) + return + } + + finalizeTask?.cancel() + asrTask?.cancel() + asr.cancel() + currentPartial = "" + lastFinal = "" + + let localeId = store.localeId + FlowSessionBridge.setTranscriptionLanguage(localeId) + FlowSessionBridge.clearPendingTranscription() + + let locale = SpeechLocaleResolver.resolve(localeId) + let stream = capture.beginUtterance() + let events = asr.transcribe(stream: stream, locale: locale) + + isUtteranceRecording = true + + asrTask = Task { @MainActor [weak self] in + guard let self else { return } + for await event in events { + switch event { + case .capability: + break + case .partial(let text): + self.currentPartial = text + case .final(let text): + self.lastFinal = text.trimmingCharacters(in: .whitespacesAndNewlines) + self.currentPartial = "" + case .error(let message): + self.debug("asr error: \(message)") + if self.isUtteranceRecording { + self.failUtterance(message: message) + } + } + } + } + + debug("utterance recording started") + } + + private func endUtterance() { + isUtteranceRecording = false + capture.endUtterance() + FlowSessionBridge.setRecordingState(.processing) + + finalizeTask?.cancel() + finalizeTask = Task { @MainActor [weak self] in + await self?.finalizeUtterance() + } + debug("utterance stopped, finalizing") + } + + private func abortUtterance() { + isUtteranceRecording = false + finalizeTask?.cancel() + asrTask?.cancel() + asr.cancel() + capture.cancelUtterance() + currentPartial = "" + lastFinal = "" + FlowSessionBridge.setRecordingState(.idle) + debug("utterance aborted") + } + + private func failUtterance(message: String) { + isUtteranceRecording = false + asrTask?.cancel() + asr.cancel() + capture.cancelUtterance() + currentPartial = "" + lastFinal = "" + FlowSessionBridge.storeTranscriptionError(message) + debug("utterance failed: \(message)") + } + + private func finalizeUtterance() async { + let deadline = Date().addingTimeInterval(30) + while Date() < deadline { + if !lastFinal.isEmpty { break } + try? await Task.sleep(nanoseconds: 100_000_000) + } + + var text = lastFinal.trimmingCharacters(in: .whitespacesAndNewlines) + if text.isEmpty { + text = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines) + } + guard !text.isEmpty else { + FlowSessionBridge.storeTranscriptionError( + NSLocalizedString("flow.error.noSpeech", comment: "") + ) + return + } + + let engineMode = store.engineMode + let modeId = store.modeId + let shouldPolish = engineMode != "local" && modeId == "polish" + + if shouldPolish { + do { + let polished = try await polisher.polish(text) + FlowSessionBridge.storeTranscriptionResult(polished) + } catch { + FlowSessionBridge.storeTranscriptionResult(text) + } + } else { + FlowSessionBridge.storeTranscriptionResult(text) + } + + currentPartial = "" + lastFinal = "" + debug("utterance finalized length=\(text.count)") + } + + // MARK: - Level publishing (main thread only) + + private func startLevelPublishing() { + levelTask?.cancel() + levelTask = Task { @MainActor [weak self] in + while !Task.isCancelled { + guard let self, self.isActive else { break } + let levels = self.capture.currentAudioLevels() + if levels.contains(where: { $0 > 0 }) { + FlowSessionBridge.storeAudioLevels(levels) + } + try? await Task.sleep(nanoseconds: 100_000_000) + } + } + } + + // MARK: - Timers + + private func startHeartbeat() { + heartbeatTask?.cancel() + FlowSessionBridge.writeHeartbeat() + heartbeatTask = Task { @MainActor [weak self] in + while !Task.isCancelled { + FlowSessionBridge.writeHeartbeat() + try? await Task.sleep(nanoseconds: 1_000_000_000) + guard self?.isActive == true else { break } + } + } + } + + private func scheduleExpiry(after duration: TimeInterval) { + expiryTask?.cancel() + expiryTask = Task { @MainActor [weak self] in + try? await Task.sleep(nanoseconds: UInt64(duration * 1_000_000_000)) + guard !Task.isCancelled else { return } + self?.endSession() + } + } + + private func debug(_ message: String) { + #if DEBUG + print("🌊[FlowSession] \(message)") + #endif + } +} diff --git a/OSGKeyboard/Services/LegalLinks.swift b/OSGKeyboard/Services/LegalLinks.swift new file mode 100644 index 0000000..5479268 --- /dev/null +++ b/OSGKeyboard/Services/LegalLinks.swift @@ -0,0 +1,11 @@ +// LegalLinks.swift +// OSGKeyboard · Main App + +import Foundation + +enum LegalLinks { + /// Public privacy policy (GitHub Pages). + static var privacyPolicyURL: URL? { + URL(string: "https://hkgood.github.io/OSGKeyboard/privacy/") + } +} diff --git a/OSGKeyboard/Services/PermissionPrimer.swift b/OSGKeyboard/Services/PermissionPrimer.swift new file mode 100644 index 0000000..357ff18 --- /dev/null +++ b/OSGKeyboard/Services/PermissionPrimer.swift @@ -0,0 +1,15 @@ +// PermissionPrimer.swift +// OSGKeyboard · Main App +// +// Legacy entry point — permission prompts now run inside Onboarding. +// Kept as a no-op so older call sites compile without re-firing TCC +// dialogs on every launch. + +import Foundation + +@MainActor +enum PermissionPrimer { + static func primeIfNeeded() async { + // Permissions are requested step-by-step in OnboardingView. + } +} diff --git a/OSGKeyboard/Views/DictationCaptureView.swift b/OSGKeyboard/Views/DictationCaptureView.swift new file mode 100644 index 0000000..48bd7f9 --- /dev/null +++ b/OSGKeyboard/Views/DictationCaptureView.swift @@ -0,0 +1,146 @@ +// DictationCaptureView.swift +// OSGKeyboard · Main App +// +// Host-app recording surface for keyboard handoff. +// Uses the shared `LiveDictationController` — the same entry point as +// the keyboard preview sheet. + +import SwiftUI +import OSGKeyboardShared + +@MainActor +final class DictationSessionCoordinator: ObservableObject { + @Published var isPresenting: Bool = false + + func present() { + isPresenting = true + } + + func dismiss() { + isPresenting = false + } +} + +struct DictationCaptureView: View { + @Environment(\.dismiss) private var dismiss + @Environment(\.themePalette) private var palette: ThemePalette + + @ObservedObject var config: ProviderConfig + @ObservedObject var coordinator: DictationSessionCoordinator + @StateObject private var dictation = LiveDictationController() + + @State private var statusText: String = "准备录音..." + @State private var isSaving: Bool = false + + var body: some View { + ZStack { + palette.background.ignoresSafeArea() + VStack(spacing: Spacing.lg) { + Spacer() + Image(systemName: "mic.circle.fill") + .font(.system(size: 84, weight: .light)) + .foregroundStyle(palette.accent) + + Text(titleText) + .font(TypeStyle.title2) + .foregroundStyle(palette.textPrimary) + + Text(statusText) + .font(TypeStyle.caption) + .foregroundStyle(palette.textSecondary) + .multilineTextAlignment(.center) + .padding(.horizontal, Spacing.lg) + + ProgressView(value: dictation.level, total: 1.0) + .tint(palette.accent) + .padding(.horizontal, Spacing.lg) + Spacer() + + HStack(spacing: Spacing.sm) { + Button { + cancelAndClose() + } label: { + Text("取消") + .secondaryButton() + } + .buttonStyle(.plain) + .disabled(isSaving) + + Button { + stopAndFinalize() + } label: { + Text("完成") + .primaryButton() + } + .buttonStyle(.plain) + .disabled(isSaving || dictation.phase != .recording) + } + .padding(.horizontal, Spacing.md) + .padding(.bottom, Spacing.lg) + } + } + .onAppear { + DictationBridge.setStatus(.requested) + startRecording() + } + .onDisappear { + dictation.stop() + } + .onChange(of: dictation.phase) { _, new in + switch new { + case .recording: + DictationBridge.setStatus(.recording) + statusText = config.isLocalEngine ? "正在实时识别..." : "正在听..." + case .processing: + DictationBridge.setStatus(.transcribing) + statusText = "处理中..." + case .requestingPermission: + statusText = "请求权限中..." + case .denied(let message): + DictationBridge.setStatus(.error, message: message) + statusText = message + case .error(let message): + DictationBridge.setStatus(.error, message: message) + statusText = message + case .idle: + break + } + } + .onChange(of: dictation.currentPartial) { _, new in + guard config.isLocalEngine, !new.isEmpty else { return } + statusText = new + } + .onChange(of: dictation.lastFinal) { _, new in + guard !new.isEmpty else { return } + saveAndClose(new) + } + } + + private var titleText: String { + isSaving ? "保存中..." : "语音输入" + } + + private func startRecording() { + Task { await dictation.start(localeId: config.localeId) } + } + + private func stopAndFinalize() { + dictation.stop() + statusText = "等待识别结果..." + } + + private func cancelAndClose() { + dictation.stop() + DictationBridge.setStatus(.cancelled) + coordinator.dismiss() + dismiss() + } + + private func saveAndClose(_ transcript: String) { + guard !isSaving else { return } + isSaving = true + DictationBridge.storePendingTranscript(transcript) + coordinator.dismiss() + dismiss() + } +} diff --git a/OSGKeyboard/Views/HomeView.swift b/OSGKeyboard/Views/HomeView.swift index 7515018..73c97a7 100644 --- a/OSGKeyboard/Views/HomeView.swift +++ b/OSGKeyboard/Views/HomeView.swift @@ -12,6 +12,7 @@ struct HomeView: View { @Environment(\.themePalette) private var palette: ThemePalette @ObservedObject var config = ProviderConfig.shared + @EnvironmentObject private var flowManager: FlowSessionManager @State private var showSettings = false @State private var showKeyboardPreview = false @@ -21,6 +22,9 @@ struct HomeView: View { VStack(spacing: 0) { statusHeader .padding(.top, Spacing.xl) + flowSessionCard + .padding(.horizontal, Spacing.md) + .padding(.top, Spacing.md) Spacer() heroButton Spacer() @@ -37,6 +41,77 @@ struct HomeView: View { } } + // MARK: - Flow session + + private var flowSessionCard: some View { + VStack(alignment: .leading, spacing: Spacing.xs) { + HStack { + Circle() + .fill(flowStatusColor) + .frame(width: 8, height: 8) + Text(flowStatusTitle) + .font(TypeStyle.caption) + .foregroundStyle(palette.textSecondary) + Spacer() + if flowManager.isActive, let expires = flowManager.sessionExpiresAt { + Text(expires, style: .timer) + .font(TypeStyle.caption2) + .foregroundStyle(palette.textTertiary) + .monospacedDigit() + } + } + Text("home.flow.hint") + .font(TypeStyle.caption2) + .foregroundStyle(palette.textTertiary) + .fixedSize(horizontal: false, vertical: true) + if let warning = flowManager.sessionWarning { + Text(warning) + .font(TypeStyle.caption2) + .foregroundStyle(palette.warning) + .fixedSize(horizontal: false, vertical: true) + if !AppPermissions.flowRequirementsMet { + Button { + AppPermissions.openSystemSettings() + } label: { + Text("home.flow.openSettings") + .font(TypeStyle.caption) + .foregroundStyle(palette.accent) + } + .buttonStyle(.plain) + } + } + if flowManager.isActive { + Button { + flowManager.endSession() + } label: { + Text("home.flow.end") + .font(TypeStyle.caption) + .foregroundStyle(palette.textSecondary) + } + .buttonStyle(.plain) + } + } + .padding(Spacing.sm) + .background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.large, style: .continuous)) + .overlay( + RoundedRectangle(cornerRadius: Radius.large, style: .continuous) + .stroke(palette.divider, lineWidth: 0.5) + ) + } + + private var flowStatusColor: Color { + if flowManager.isActive { return palette.success } + if flowManager.isStarting { return palette.accent } + if flowManager.sessionWarning != nil { return palette.warning } + return palette.warning + } + + private var flowStatusTitle: LocalizedStringKey { + if flowManager.isActive { return "home.flow.active" } + if flowManager.isStarting { return "home.flow.starting" } + return "home.flow.inactive" + } + // MARK: - Header private var statusHeader: some View { diff --git a/OSGKeyboard/Views/KeyboardPreviewSheet.swift b/OSGKeyboard/Views/KeyboardPreviewSheet.swift index c98fe86..5bebf61 100644 --- a/OSGKeyboard/Views/KeyboardPreviewSheet.swift +++ b/OSGKeyboard/Views/KeyboardPreviewSheet.swift @@ -1,82 +1,32 @@ // KeyboardPreviewSheet.swift // OSGKeyboard · Main App (Debug) // -// In-app preview of the keyboard extension. Renders a stand-in -// `KeyboardPreviewStub` so the user can see what the real extension -// looks like, AND drives a *real* `PreviewASRController` so tapping -// the disc actually records from the mic and runs the shared ASR -// service (`SpeechAnalyzer` on iOS 26+), -// and lands recognized text in the top textbox. Without the real ASR -// the preview was a static mock — "the text never appears" was a -// fair review note. +// In-app preview of the keyboard extension. Drives the shared +// `LiveDictationController` so tapping record exercises the same +// on-device ASR pipeline as host-app dictation handoff. // -// Lifecycle (local engine = "transcribe" only, no LLM): -// tap → start ASR (idempotent re-entry guard) -// → ASR emits .partial / .final -// → currentPartial updates the transcript line in real time -// tap → stop ASR -// → lastFinal event lands -// → onChange in this view appends to typedText -// → controller resets +// Local engine: partial transcripts stream into the text box live. +// Cloud engine: final transcript is appended when recording stops. import SwiftUI import OSGKeyboardShared struct KeyboardPreviewSheet: View { @Environment(\.themePalette) private var palette: ThemePalette - @Environment(\.dismiss) private var dismiss - @ObservedObject private var config = ProviderConfig.shared - @StateObject private var asr = PreviewASRController() + @StateObject private var dictation = LiveDictationController() @State private var showSettings = false - /// Accumulates text in the top textbox. The real keyboard extension - /// inserts directly via `textDocumentProxy`; this preview mirrors - /// that in a parallel `@State` so the user can verify the flow. @State private var typedText: String = "" - /// Cached snapshot of the previous lastFinal so the .onChange - /// doesn't fire on every body re-render — only on real changes. @State private var lastFinalSeen: String = "" - - private enum StubPhase { case idle, recording, processing } - - /// Visible phase for the stub. Driven by the real ASR controller - /// — when the controller is `recording` we show the recording - /// state; otherwise we show `processing` while a final is in - /// flight and `idle` otherwise. - private var stubPhase: KeyboardPreviewStub.Phase { - switch asr.phase { - case .recording: return .recording - case .processing: return .processing - case .idle: return .idle - case .requestingPermission: - return .recording - case .denied, .error: - return .idle - } - } - - /// The transcript line the stub shows under the chips. While - /// recording we surface the live ASR partial; otherwise we surface - /// any error or stay quiet. - private var stubTranscript: String { - switch asr.phase { - case .recording: - return asr.currentPartial.isEmpty ? " " : asr.currentPartial - case .error(let m): - return m - case .denied(let m): - return m - default: - return "" - } - } + /// Text in the box when the current dictation session started. + @State private var dictationAnchorText: String = "" var body: some View { ZStack { palette.background.ignoresSafeArea() - VStack(spacing: 0) { + VStack(spacing: Spacing.md) { VStack(spacing: Spacing.md) { Text("preview.title") .font(TypeStyle.title2) @@ -87,55 +37,62 @@ struct KeyboardPreviewSheet: View { .multilineTextAlignment(.center) .padding(.horizontal, Spacing.lg) mockTextField.padding(.horizontal, Spacing.md) + controls + .padding(.horizontal, Spacing.md) } .padding(.top, Spacing.lg) - Spacer(minLength: 0) - keyboardBlock + Spacer() } } .sheet(isPresented: $showSettings) { SettingsView() } - .onChange(of: asr.lastFinal) { _, new in + .onChange(of: dictation.currentPartial) { _, partial in + guard config.isLocalEngine, isDictationActive else { return } + applyLiveTranscript(partial) + } + .onChange(of: dictation.lastFinal) { _, new in guard !new.isEmpty, new != lastFinalSeen else { return } lastFinalSeen = new - insertRecognizedText(new) - asr.reset() + if config.isLocalEngine { + applyLiveTranscript(new) + dictationAnchorText = typedText + } else { + insertRecognizedText(new) + } + dictation.reset() } - // Tear down the ASR pipeline when the sheet leaves the screen - // (preview dismissed, app backgrounded mid-recording, etc.). - // Without this, a leftover `asrTask` keeps the AVAudioSession - // active and the mic permission in use after the user has - // moved on. `asr.stop()` is idempotent — it no-ops on - // `.idle`/`.denied`/`.error` — so it's safe to call here - // even when the disc is not currently recording. .onDisappear { - asr.stop() + dictation.stop() } } - /// Real `TextField` (was a static placeholder HStack before the - /// first fix). The user can both type into it AND see recognized - /// text land in it as the real ASR fires `.final`. private var mockTextField: some View { - HStack(spacing: Spacing.xs) { - Image(systemName: "text.cursor") - .foregroundStyle(palette.textSecondary) - TextField(LocalizedStringKey("preview.placeholder"), text: $typedText, axis: .vertical) - .lineLimit(1...4) - .textFieldStyle(.plain) + VStack(alignment: .leading, spacing: Spacing.xs) { + HStack { + Image(systemName: "text.cursor") + .foregroundStyle(palette.textSecondary) + Text(statusTitle) + .font(TypeStyle.caption) + .foregroundStyle(palette.textSecondary) + Spacer() + if !typedText.isEmpty { + Button { + typedText = "" + } label: { + Image(systemName: "xmark.circle.fill") + .foregroundStyle(palette.textTertiary) + } + .buttonStyle(.plain) + .accessibilityLabel("preview.clear") + } + } + TextEditor(text: $typedText) + .frame(minHeight: 260) + .scrollContentBackground(.hidden) .foregroundStyle(palette.textPrimary) .tint(palette.accent) - if !typedText.isEmpty { - Button { - typedText = "" - } label: { - Image(systemName: "xmark.circle.fill") - .foregroundStyle(palette.textTertiary) - } - .buttonStyle(.plain) - .accessibilityLabel("preview.clear") - } + .padding(.horizontal, 2) } .padding(Spacing.md) .background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.medium)) @@ -145,94 +102,99 @@ struct KeyboardPreviewSheet: View { ) } - private var keyboardBlock: some View { - VStack(spacing: 0) { - Rectangle() - .fill(palette.divider) - .frame(height: 0.5) - KeyboardPreviewStub( - phase: stubPhase, - level: asr.level, - transcript: stubTranscript, - modeId: config.modeId, - localeId: config.localeId, - onTap: cyclePhase, - openSettings: { showSettings = true }, - onModeCycle: cycleMode, - onLocaleCycle: cycleLocale - ) + private var controls: some View { + VStack(spacing: Spacing.xs) { + HStack(spacing: Spacing.sm) { + Button { + toggleRecording() + } label: { + Label(isRecording ? "停止录音" : "开始录音", systemImage: isRecording ? "stop.circle.fill" : "mic.circle.fill") + .primaryButton() + } + .buttonStyle(.plain) + + Button { + showSettings = true + } label: { + Label("设置", systemImage: "gearshape") + .secondaryButton() + } + .buttonStyle(.plain) + } + + Text(serviceLabel) + .font(TypeStyle.caption2) + .foregroundStyle(palette.textTertiary) + .frame(maxWidth: .infinity, alignment: .center) + .lineLimit(2) + .multilineTextAlignment(.center) } } - /// Tap on the disc. Drives the *real* ASR pipeline — the previous - /// mock that just toggled a hardcoded phase is gone. - private func cyclePhase() { + private var serviceLabel: String { + EngineServiceLabel.summary( + engineMode: config.engineMode, + providerId: config.providerId, + model: config.model + ) + } + + private var isRecording: Bool { + if case .recording = dictation.phase { return true } + return false + } + + private var isDictationActive: Bool { + switch dictation.phase { + case .recording, .processing: return true + default: return false + } + } + + private var statusTitle: String { + switch dictation.phase { + case .idle: return "输入框" + case .requestingPermission: return "请求权限中..." + case .recording: return config.isLocalEngine ? "正在实时识别..." : "正在录音..." + case .processing: return "识别中..." + case .denied(let message), .error(let message): + return message + } + } + + private func toggleRecording() { withAnimation(Motion.quick) { - switch asr.phase { + switch dictation.phase { case .idle, .denied, .error: - let locale = resolveLocale(config.localeId) - Task { await asr.start(locale: locale) } + startRecording() case .recording: - asr.stop() + stopAndFinalize() case .requestingPermission, .processing: break } } } - /// Cycle the input mode on tap of the mode chip. The order mirrors - /// the Settings picker (`off` → `transcribe` → `polish` → wrap) - /// so the user sees the same surface in both places. - /// - /// Note: when the user picks `off` we *also* stop any in-flight - /// recording — leaving the disc mid-recording in an "off" mode - /// would be a confusing state (the user is recording but the - /// keyboard says it won't insert anything). - private func cycleMode() { - let order = ["off", "transcribe", "polish"] - let current = order.firstIndex(of: config.modeId) ?? 0 - let next = order[(current + 1) % order.count] - config.modeId = next - if next == "off" && asr.phase == .recording { - asr.stop() - } + private func startRecording() { + dictationAnchorText = typedText + lastFinalSeen = "" + Task { await dictation.start(localeId: config.localeId) } } - /// Cycle the locale on tap of the locale chip. Same order as - /// `staticLocales` in `SettingsView.swift` so both surfaces stay - /// in sync. When the user picks a new locale we *also* stop any - /// in-flight recording — ASR sessions are bound to the locale - /// they were started with, and continuing to feed buffers into a - /// stale session would produce garbage in the next `.final`. - private func cycleLocale() { - let order = ["auto", "zh-Hans", "zh-Hant", "en-US", "ja-JP", "ko-KR"] - let current = order.firstIndex(of: config.localeId) ?? 0 - let next = order[(current + 1) % order.count] - config.localeId = next - if asr.phase == .recording { - asr.stop() - } + private func stopAndFinalize() { + dictation.stop() } - private func resolveLocale(_ id: String) -> Locale { - if id == "auto" { return .current } - return Locale(identifier: id) + private func applyLiveTranscript(_ transcript: String) { + let composed = DictationTextComposer.compose(anchor: dictationAnchorText, live: transcript) + guard composed != typedText else { return } + typedText = composed } - /// Append the recognized transcript to the textbox, with a leading - /// space when the existing text doesn't already end in whitespace. - /// Matches what `textDocumentProxy.insertText` does for the real - /// keyboard when the user's draft has no trailing whitespace. private func insertRecognizedText(_ recognized: String) { let trimmed = recognized.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return } - if typedText.isEmpty { - typedText = trimmed - } else if typedText.last == " " || typedText.last == "\n" { - typedText += trimmed - } else { - typedText += " " + trimmed - } + typedText = DictationTextComposer.compose(anchor: typedText, live: trimmed) } } diff --git a/OSGKeyboard/Views/OnboardingView.swift b/OSGKeyboard/Views/OnboardingView.swift index 4e3539c..1f94790 100644 --- a/OSGKeyboard/Views/OnboardingView.swift +++ b/OSGKeyboard/Views/OnboardingView.swift @@ -1,32 +1,26 @@ // OnboardingView.swift // OSGKeyboard · Main App // -// Three-step onboarding: -// -// 1) Welcome — what the app does, in one sentence -// 2) Enable — Settings → General → Keyboards → Add → Allow Full Access -// 3) Setup — pick a provider, paste a key -// -// We deliberately do NOT use `TabView` with `.page` style for the -// pager. That style wraps the content in a `UIPageViewController`, -// and `UIPageViewController` has a long-standing behavior on some iOS -// versions where -// the keyboard-showing layout reflow on a `TextField` focus is -// misread as a horizontal swipe — the page jumps back to step 1 -// the moment the user starts typing. Replacing the TabView with a -// `ZStack`-based conditional view sidesteps the bug entirely; we -// give up the swipe-to-page gesture, but the Back/Next buttons at -// the bottom (and the page dots) are the canonical onboarding -// affordance and the user is never more than one tap from the next -// page anyway. -// -// Visual style: one large accent surface, generous whitespace, single CTA -// at the bottom. No tipsy animations, no cheerful illustrations — every -// pixel is doing one job. +// Five-step onboarding: +// 1) Welcome +// 2) Microphone permission +// 3) Speech recognition permission +// 4) Enable keyboard + Allow Full Access +// 5) Engine / API setup import SwiftUI import OSGKeyboardShared +private enum OnboardingPage: Int, CaseIterable { + case welcome = 0 + case microphone + case speech + case keyboard + case api + + static let count = 5 +} + struct OnboardingView: View { @Environment(\.themePalette) private var palette: ThemePalette @@ -38,10 +32,12 @@ struct OnboardingView: View { palette.background.ignoresSafeArea() VStack(spacing: 0) { Group { - switch page { - case 0: WelcomePage() - case 1: EnableKeyboardPage() - default: APISetupPage(config: config) + switch OnboardingPage(rawValue: page) ?? .welcome { + case .welcome: WelcomePage() + case .microphone: MicPermissionPage() + case .speech: SpeechPermissionPage() + case .keyboard: EnableKeyboardPage() + case .api: APISetupPage(config: config) } } .frame(maxWidth: .infinity, maxHeight: .infinity) @@ -59,7 +55,7 @@ struct OnboardingView: View { private var pageDots: some View { HStack(spacing: 6) { - ForEach(0..<3, id: \.self) { i in + ForEach(0.. some View { HStack(alignment: .top, spacing: Spacing.xs) { Image(systemName: icon) @@ -192,7 +200,180 @@ private struct PrivacyFootnote: View { } } -// MARK: - Page 2: Enable keyboard +// MARK: - Permission pages + +private struct MicPermissionPage: View { + @Environment(\.themePalette) private var palette: ThemePalette + @State private var status = AppPermissions.micStatus + @State private var isRequesting = false + + var body: some View { + PermissionPageLayout( + icon: "mic.fill", + title: "onboarding.permission.mic.title", + detail: "onboarding.permission.mic.body", + status: statusLabel, + statusColor: statusColor, + primaryTitle: primaryButtonTitle, + primaryDisabled: isRequesting || status == .granted, + onPrimary: { Task { await request() } }, + secondaryTitle: status == .denied ? "onboarding.permission.openSettings" : nil, + onSecondary: status == .denied ? { AppPermissions.openSystemSettings() } : nil, + deniedHint: status == .denied ? "onboarding.permission.mic.deniedHint" : nil + ) + .onAppear { status = AppPermissions.micStatus } + } + + private var statusLabel: LocalizedStringKey { + switch status { + case .undetermined: return "onboarding.permission.status.undetermined" + case .granted: return "onboarding.permission.status.granted" + case .denied: return "onboarding.permission.status.denied" + } + } + + private var statusColor: Color { + switch status { + case .granted: return palette.success + case .denied: return palette.warning + case .undetermined: return palette.textTertiary + } + } + + private var primaryButtonTitle: LocalizedStringKey { + status == .granted ? "onboarding.permission.status.granted" : "onboarding.permission.mic.allow" + } + + private func request() async { + isRequesting = true + _ = await AppPermissions.requestMicrophone() + status = AppPermissions.micStatus + isRequesting = false + } +} + +private struct SpeechPermissionPage: View { + @Environment(\.themePalette) private var palette: ThemePalette + @State private var isRequesting = false + + var body: some View { + PermissionPageLayout( + icon: "waveform.badge.mic", + title: "onboarding.permission.speech.title", + detail: "onboarding.permission.speech.body", + status: statusLabel, + statusColor: statusColor, + primaryTitle: primaryButtonTitle, + primaryDisabled: isRequesting || AppPermissions.speechStatus == .granted, + onPrimary: { Task { await request() } }, + secondaryTitle: speechDenied ? "onboarding.permission.openSettings" : nil, + onSecondary: speechDenied ? { AppPermissions.openSystemSettings() } : nil, + deniedHint: speechDenied ? "onboarding.permission.speech.deniedHint" : nil + ) + } + + private var speechDenied: Bool { + switch AppPermissions.speechStatus { + case .denied, .restricted: return true + default: return false + } + } + + private var statusLabel: LocalizedStringKey { + switch AppPermissions.speechStatus { + case .undetermined: return "onboarding.permission.status.undetermined" + case .granted: return "onboarding.permission.status.granted" + case .denied, .restricted: return "onboarding.permission.status.denied" + } + } + + private var statusColor: Color { + switch AppPermissions.speechStatus { + case .granted: return palette.success + case .denied, .restricted: return palette.warning + case .undetermined: return palette.textTertiary + } + } + + private var primaryButtonTitle: LocalizedStringKey { + AppPermissions.speechStatus == .granted + ? "onboarding.permission.status.granted" + : "onboarding.permission.speech.allow" + } + + private func request() async { + isRequesting = true + _ = await AppPermissions.requestSpeechRecognition() + isRequesting = false + } +} + +private struct PermissionPageLayout: View { + @Environment(\.themePalette) private var palette: ThemePalette + + let icon: String + let title: LocalizedStringKey + let detail: LocalizedStringKey + let status: LocalizedStringKey + let statusColor: Color + let primaryTitle: LocalizedStringKey + let primaryDisabled: Bool + let onPrimary: () -> Void + var secondaryTitle: LocalizedStringKey? = nil + var onSecondary: (() -> Void)? = nil + var deniedHint: LocalizedStringKey? = nil + + var body: some View { + VStack(spacing: Spacing.lg) { + Spacer() + Image(systemName: icon) + .font(.system(size: 64, weight: .light)) + .foregroundStyle(palette.accent) + VStack(spacing: Spacing.sm) { + Text(title) + .font(TypeStyle.title2) + .foregroundStyle(palette.textPrimary) + Text(detail) + .font(TypeStyle.body) + .foregroundStyle(palette.textSecondary) + .multilineTextAlignment(.center) + .padding(.horizontal, Spacing.lg) + } + HStack(spacing: 6) { + Circle().fill(statusColor).frame(width: 8, height: 8) + Text(status) + .font(TypeStyle.caption) + .foregroundStyle(palette.textSecondary) + } + if let deniedHint { + Text(deniedHint) + .font(TypeStyle.caption2) + .foregroundStyle(palette.warning) + .multilineTextAlignment(.center) + .padding(.horizontal, Spacing.lg) + } + VStack(spacing: Spacing.xs) { + Button(action: onPrimary) { + Text(primaryTitle) + .primaryButton() + } + .buttonStyle(.plain) + .disabled(primaryDisabled) + if let secondaryTitle, let onSecondary { + Button(action: onSecondary) { + Text(secondaryTitle) + .secondaryButton() + } + .buttonStyle(.plain) + } + } + .padding(.horizontal, Spacing.md) + Spacer() + } + } +} + +// MARK: - Enable keyboard private struct EnableKeyboardPage: View { @Environment(\.themePalette) private var palette: ThemePalette @@ -207,6 +388,11 @@ private struct EnableKeyboardPage: View { Text("onboarding.enable.title") .font(TypeStyle.title2) .foregroundStyle(palette.textPrimary) + Text("onboarding.enable.fullAccessNote") + .font(TypeStyle.caption) + .foregroundStyle(palette.textSecondary) + .multilineTextAlignment(.center) + .padding(.horizontal, Spacing.lg) } VStack(alignment: .leading, spacing: Spacing.sm) { step(num: 1, text: NSLocalizedString("onboarding.enable.step1", comment: "")) @@ -218,13 +404,12 @@ private struct EnableKeyboardPage: View { .padding(.horizontal, Spacing.md) Button { - if let url = URL(string: UIApplication.openSettingsURLString) { - UIApplication.shared.open(url) - } + AppPermissions.openSystemSettings() } label: { Label(LocalizedStringKey("onboarding.enable.openSettings"), systemImage: "arrow.up.right.square") .primaryButton() } + .buttonStyle(.plain) .padding(.horizontal, Spacing.md) Spacer() } @@ -245,7 +430,7 @@ private struct EnableKeyboardPage: View { } } -// MARK: - Page 3: API setup +// MARK: - API setup private struct APISetupPage: View { @Environment(\.themePalette) private var palette: ThemePalette @@ -267,20 +452,15 @@ private struct APISetupPage: View { .padding(.horizontal, Spacing.md) .padding(.top, Spacing.lg) - // Same Engine picker as Settings — see EnginePickerSection. EnginePickerSection(config: config) .padding(.horizontal, Spacing.md) if config.engineMode == "cloud" { ProviderPickerSection(config: config) .padding(.horizontal, Spacing.md) - APISettingsCard(config: config) .padding(.horizontal, Spacing.md) } else { - // Local path: no LLM, no API key needed. Show a short - // confirmation so the user understands "no further - // setup required". VStack(alignment: .leading, spacing: Spacing.xs) { HStack(spacing: Spacing.sm) { Image(systemName: "checkmark.seal.fill") diff --git a/OSGKeyboard/Views/PreviewASRController.swift b/OSGKeyboard/Views/PreviewASRController.swift index 92bb195..88f9c85 100644 --- a/OSGKeyboard/Views/PreviewASRController.swift +++ b/OSGKeyboard/Views/PreviewASRController.swift @@ -1,462 +1,9 @@ // PreviewASRController.swift -// OSGKeyboard · Main App (Debug) +// OSGKeyboard · Main App // -// Self-contained ASR controller for the in-app keyboard preview sheet. -// Owns its own AVAudioEngine + AVAudioSession, downsamples to 16 kHz -// mono Float32, and feeds the `AudioBufferSnapshot` stream to the -// shared `ASRService` (the same pipeline the real keyboard extension -// uses, so the preview exercises the *real* iOS speech APIs, not a -// stub). Without this the in-app preview was a hardcoded transcript -// and "did you actually call SFSpeechRecognizer?" was a fair review -// note. -// -// Why not reuse `AudioCaptureService` from the extension? It lives in -// `OSGKeyboardExt`, an `app-extension` target — the main app can't -// import its symbols. We could move it to `OSGKeyboardShared`, but -// `AVAudioSession` lifecycle differs enough between a keyboard -// extension (no background, no recording entitlement surprise) and a -// foreground app that a copy here is the lesser evil. +// Legacy name kept for existing call sites and tests. The implementation +// lives in `OSGKeyboardShared` as `LiveDictationController`. -import Foundation -import AVFoundation -import Speech import OSGKeyboardShared -@MainActor -final class PreviewASRController: ObservableObject { - - enum Phase: Equatable { - case idle - case requestingPermission - case recording - case processing - case denied(String) - case error(String) - } - - @Published private(set) var phase: Phase = .idle - /// Normalized 0...1 RMS for the disc level meter. Polled from the - /// audio tap via `Task { @MainActor in ... }` — the tap itself - /// runs on a real-time audio thread, so we never touch published - /// state from there. - @Published private(set) var level: Double = 0 - @Published private(set) var currentPartial: String = "" - @Published private(set) var errorMessage: String? - - /// Set when a `.final` ASR event lands. The owning sheet observes - /// this and appends the text to its textbox, then clears it so the - /// next recording starts from zero. - @Published var lastFinal: String = "" - - private let asr: ASRService = ASRServiceFactory.make() - private let audioEngine = AVAudioEngine() - /// `internal` (not `private`) so the regression test in - /// `OSGKeyboardTests/PreviewASRControllerStateTests.swift` can - /// install a known consumer task and assert `stop()` doesn't - /// cancel it. The class is `@MainActor`-isolated, so the - /// natural Swift 6 isolation rules still prevent production - /// code outside the class from racing on it. - var asrTask: Task? - private var bufferContinuation: AsyncStream.Continuation? - private var didConfigureAudioSession = false - private var didInstallTap = false - - func start(locale: Locale) async { - // Re-entry guard: ignore taps that arrive while we're already - // running. (The sheet's `cyclePhase` is also guarded, but - // async race windows are easier to lock down here.) - switch phase { - case .recording, .requestingPermission, .processing: - return - default: - break - } - // Cancel any leftover consumer task from a previous recording. - // Normally `stop()` lets the task run to completion (so it can - // see the `.final` and transition out of `.processing`), but if - // the user smashed the disc twice — stop, then immediately - // start — the previous task might still be draining. Cancel it - // here so we don't have two consumer tasks fighting over the - // same `events` stream. - asrTask?.cancel() - asrTask = nil - phase = .requestingPermission - currentPartial = "" - lastFinal = "" - errorMessage = nil - level = 0 - - // 1. Microphone permission. The helper is `nonisolated` so the - // (iOS < 17) callback closure does not inherit `@MainActor` — - // `AVAudioSession.requestRecordPermission` delivers on a TCC - // reply queue, and a `@MainActor`-inferred closure body there - // hits `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift`. - let micGranted = await Self.requestMicrophonePermission() - guard micGranted else { - phase = .denied(NSLocalizedString("keyboard.denied.mic", comment: "")) - return - } - - // 2. Speech recognition permission. Same reasoning as above: - // the callback fires on TCC's reply queue, NOT the main queue. - let speechGranted = await Self.requestSpeechRecognitionPermission() - guard speechGranted else { - phase = .denied(NSLocalizedString("keyboard.denied.speech", comment: "")) - return - } - - // 3. Audio session — only configure once per process. - // - // Category is `.record` (not `.playAndRecord`) because the - // preview never plays back audio — it just records from the - // mic and hands the buffers to `SpeechAnalyzer`. On the - // iOS Simulator, `.playAndRecord` requires the - // `AURemoteIO` Audio Unit's *output* side to also be - // enabled, but the simulator's "speaker" reports a 0 Hz - // hardware format, so `AURemoteIO::enable` fails with - // `kAudioUnitErr_FormatNotSupported` (-10851) and any - // subsequent `installTap` traps with "Failed to create tap - // due to format mismatch". `.record` skips the output - // side entirely, so the simulator can record. - // - // The real keyboard extension (`OSGKeyboardExt`) keeps - // `.playAndRecord` because it runs on a real device where - // the output side has a real hardware format, and may want - // to play click sounds / haptic feedback. Only the preview - // needs the simulator-friendly category. - if !didConfigureAudioSession { - do { - let session = AVAudioSession.sharedInstance() - try session.setCategory(.record, - mode: .measurement, - options: []) - try session.setActive(true, options: .notifyOthersOnDeactivation) - didConfigureAudioSession = true - } catch { - phase = .error(String.localizedStringWithFormat( - NSLocalizedString("preview.error.audioSession", comment: ""), - error.localizedDescription - )) - return - } - } - - // 4. Spin up the engine + ASR. - phase = .recording - startEngineAndASR(locale: locale) - } - - func stop() { - // Don't `asrTask?.cancel()` here — see the comment in - // `startEngineAndASR` for the full rationale. Short version: - // cancelling the consumer task at the same moment we close the - // audio stream also triggers the producer's - // `continuation.onTermination → self?.cancel()` cascade, which - // marks the producer's outer task as cancelled and skips the - // `.final` event. The UI is then left in `.processing` forever - // because no one schedules the transition out. The consumer - // task naturally exits when `events` finishes, so the right - // thing is to let it run. - // - // If a previous `asrTask` is somehow still running (e.g. the - // user smashed the disc twice quickly), `start()` cancels it - // at the entry point as a safety net. - if didInstallTap { - audioEngine.inputNode.removeTap(onBus: 0) - didInstallTap = false - } - if audioEngine.isRunning { - audioEngine.stop() - } - bufferContinuation?.finish() - bufferContinuation = nil - if phase == .recording { - phase = .processing - } - // Deactivate so the user's music resumes if the preview is - // dismissed mid-recording. - try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) - - // Safety net: if the ASR pipeline never produces a `.final` - // (analyzer hang, system glitch, dropped continuation), force - // the UI back to idle after a short delay so the user isn't - // stuck. Normal recordings complete well under a second, so - // the 3-second budget is only hit on the unhappy path; if the - // pipeline finishes first and flips the phase to `.idle` (or - // `.error`), the check below no-ops. - Task { @MainActor [weak self] in - try? await Task.sleep(for: .seconds(3)) - guard let self else { return } - if self.phase == .processing { - self.phase = .idle - } - } - } - - func reset() { - // Called by the sheet after appending `lastFinal` to the textbox, - // so the next recording can produce a fresh final without us - // double-appending. - lastFinal = "" - if phase == .processing { - phase = .idle - } - } - - // MARK: - Engine + ASR - - private func startEngineAndASR(locale: Locale) { - let inputNode = audioEngine.inputNode - let hwFormat = inputNode.outputFormat(forBus: 0) - - // Pre-flight check: a placeholder / unconfigured input bus - // reports `sampleRate == 0` (or `channelCount == 0`). - // `installTap` on such a bus traps with "Failed to create - // tap due to format mismatch" (an NSException, not a Swift - // `Error`, so we can't `try`/`catch` it). The safest fix - // is to refuse the tap up front and surface a clear - // `.error` phase instead of crashing the app. We've seen - // this on the iOS Simulator when the host's microphone - // permission isn't granted to CoreSimulator, and on - // devices where the audio session is in an unexpected - // state from a previous foreground/background transition. - guard hwFormat.sampleRate > 0, hwFormat.channelCount > 0 else { - phase = .error( - String.localizedStringWithFormat( - NSLocalizedString("preview.error.micUnavailable", comment: ""), - hwFormat.sampleRate, - Int(hwFormat.channelCount) - ) - ) - return - } - - let targetSampleRate: Double = 16_000 - guard let targetFormat = AVAudioFormat( - commonFormat: .pcmFormatFloat32, - sampleRate: targetSampleRate, - channels: 1, - interleaved: false - ) else { - phase = .error(NSLocalizedString("preview.error.formatCreate", comment: "")) - return - } - guard let converter = AVAudioConverter(from: hwFormat, to: targetFormat) else { - phase = .error(NSLocalizedString("preview.error.converterCreate", comment: "")) - return - } - - let (stream, continuation) = AsyncStream.makeStream() - self.bufferContinuation = continuation - - // Tap the hardware input. The closure passed to `installTap` runs - // on the AVAudioEngine real-time audio thread. In Swift 6 strict - // concurrency, a closure literal defined inside a `@MainActor` - // method inherits `@MainActor` isolation, which would trip - // `dispatch_assert_queue_fail` on first invocation from the - // audio thread. The fix is to build the actual tap body in a - // `nonisolated` helper (`makeAudioTapBlock`) and have the - // installTap closure be a single function reference — function - // references never carry inferred isolation, so the dispatch - // runtime is happy and the body runs wherever AVAudioEngine - // wants it (the audio thread). - let onMeter: @Sendable (Double) -> Void = { [weak self] meter in - Task { @MainActor [weak self] in - guard let self else { return } - // Lightweight smoothing so the disc ring doesn't jitter. - self.level = self.level * 0.55 + meter * 0.45 - } - } - let onSnapshot: @Sendable (AudioBufferSnapshot) -> Void = { [weak self] snapshot in - Task { @MainActor [weak self] in - self?.bufferContinuation?.yield(snapshot) - } - } - let tap = Self.makeAudioTapBlock( - converter: converter, - targetFormat: targetFormat, - hwFormat: hwFormat, - targetSampleRate: targetSampleRate, - onMeter: onMeter, - onSnapshot: onSnapshot - ) - inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap) - didInstallTap = true - - audioEngine.prepare() - do { - try audioEngine.start() - } catch { - phase = .error(String.localizedStringWithFormat( - NSLocalizedString("preview.error.engineStart", comment: ""), - error.localizedDescription - )) - return - } - - // 5. Wire up ASR. - let events = asr.transcribe( - stream: stream, - locale: locale - ) - asrTask = Task { @MainActor [weak self] in - guard let self else { return } - for await event in events { - switch event { - case .capability: - // Could surface on-device vs cloud here; preview - // doesn't need it. - break - case .partial(let s): - self.currentPartial = s - case .final(let s): - let trimmed = s.trimmingCharacters(in: .whitespacesAndNewlines) - self.lastFinal = trimmed - self.currentPartial = "" - if !trimmed.isEmpty { - self.phase = .idle - } else { - // Empty final: nothing recognized. Return to idle - // without triggering a textbox insert. - self.phase = .idle - } - case .error(let m): - self.errorMessage = m - self.phase = .error(m) - } - } - } - } - - // MARK: - Permission helpers (nonisolated) - // - // `SFSpeechRecognizer.requestAuthorization` delivers its callback - // on a TCC reply queue, NOT the main queue. If we wrap that - // callback inline in `start(locale:)` — which is `@MainActor` — - // Swift 6 strict concurrency infers the closure body as - // `@MainActor`, and the runtime crashes on - // `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift` as - // soon as TCC calls us back. - // - // The first attempt (commit `e8a0310`) extracted the entire - // permission request into a `nonisolated static func` helper. - // That worked in isolation, but the Swift 6 optimizer - // inlined those helpers back into `start(locale:)`. After - // inlining, the `withCheckedContinuation` body and the - // `requestAuthorization` callback were re-typed in the - // `@MainActor` context of the caller, and the runtime - // assertion came right back — same crash, different symbol: - // `closure #1 in closure #2 in PreviewASRController.start(locale:)`. - // - // The fix that survives inlining is the *function-reference* - // pattern, the same one used for `installTap` in - // `makeAudioTapBlock` below. The callback is built in a - // `nonisolated` static helper that takes a `CheckedContinuation` - // and returns the `(Status) -> Void` handler. The body of that - // helper has no enclosing actor, so the closure is created in - // nonisolated context. When TCC calls us back, the runtime - // sees a nonisolated closure on a non-main queue and is happy. - // - // `cont.resume(...)` is itself thread-safe on - // `CheckedContinuation`, so we don't need to hop back to the - // main actor before resuming. - - private nonisolated static func requestMicrophonePermission() async -> Bool { - // iOS 17+ API; the iOS < 17 fallback (`AVAudioSession.recordPermission` - // + `requestRecordPermission` callback) is gone now that the - // deployment target is iOS 26. - switch AVAudioApplication.shared.recordPermission { - case .granted: return true - case .denied: return false - case .undetermined: return await AVAudioApplication.requestRecordPermission() - @unknown default: return false - } - } - - private nonisolated static func requestSpeechRecognitionPermission() async -> Bool { - await withCheckedContinuation { (cont: CheckedContinuation) in - SFSpeechRecognizer.requestAuthorization( - Self.makeSpeechAuthHandler(continuation: cont) - ) - } - } - - private nonisolated static func makeSpeechAuthHandler( - continuation: CheckedContinuation - ) -> @Sendable (SFSpeechRecognizerAuthorizationStatus) -> Void { - return { status in - continuation.resume(returning: status == .authorized) - } - } - - // MARK: - Audio tap (nonisolated, runs on AVAudioEngine render thread) - // - // `AVAudioNode.installTap`'s callback fires on the audio engine's - // real-time render thread. In Swift 6 strict concurrency, a closure - // literal defined inside a `@MainActor` method inherits `@MainActor` - // isolation — and `dispatch_assert_queue_fail` fires the moment - // the runtime tries to dispatch that closure on a non-main queue. - // - // The trick is to build the actual tap body in a `nonisolated` - // function and have the installTap closure be a *function reference* - // to that helper. Function references never carry inferred - // isolation, so the dispatch runtime is satisfied and the body - // runs wherever AVAudioEngine wants. State updates to - // `self.level` and the AsyncStream continuation hop back to the - // main actor via `Task { @MainActor in … }`, which is itself - // safe to call from a non-isolated context. - private nonisolated static func makeAudioTapBlock( - converter: AVAudioConverter, - targetFormat: AVAudioFormat, - hwFormat: AVAudioFormat, - targetSampleRate: Double, - onMeter: @Sendable @escaping (Double) -> Void, - onSnapshot: @Sendable @escaping (AudioBufferSnapshot) -> Void - ) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void { - // `@Sendable` on the returned closure makes the Sendable - // conformance explicit. `AVAudioNodeTapBlock` is declared as - // a plain escaping closure in the SDK; we cast at the call - // site via `as @Sendable`. - return { buffer, _ in - // Downsample + extract samples + compute RMS in one pass. - let ratio = targetSampleRate / hwFormat.sampleRate - let outCapacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio + 0.5) - guard outCapacity > 0, - let converted = AVAudioPCMBuffer( - pcmFormat: targetFormat, - frameCapacity: outCapacity - ) else { return } - - var error: NSError? - var supplied = false - converter.convert(to: converted, error: &error) { _, outStatus in - if supplied { - outStatus.pointee = .endOfStream - return nil - } - supplied = true - outStatus.pointee = .haveData - return buffer - } - if error != nil { return } - - let n = Int(converted.frameLength) - var samples = [Float](repeating: 0, count: n) - var sumSquares: Float = 0 - if let channelData = converted.floatChannelData?[0] { - for i in 0.. 0 ? sqrtf(sumSquares / Float(n)) : 0 - // RMS for speech is typically 0.02-0.2; the 4x gain pushes - // normal speech into the 0.4-0.8 range for the disc meter. - let meter = min(Double(rms) * 4.0, 1.0) - let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: targetSampleRate) - onMeter(meter) - onSnapshot(snapshot) - } - } -} +typealias PreviewASRController = LiveDictationController diff --git a/OSGKeyboard/Views/SettingsView.swift b/OSGKeyboard/Views/SettingsView.swift index 561af6f..a5d475b 100644 --- a/OSGKeyboard/Views/SettingsView.swift +++ b/OSGKeyboard/Views/SettingsView.swift @@ -33,6 +33,7 @@ struct SettingsView: View { if config.engineMode == "cloud" { promptSection } + privacySection resetButton } .padding(.horizontal, Spacing.md) @@ -253,6 +254,39 @@ struct SettingsView: View { } } + // MARK: - Privacy + + private var privacySection: some View { + VStack(alignment: .leading, spacing: Spacing.xs) { + sectionHeader("settings.privacy.title", subtitle: nil) + VStack(alignment: .leading, spacing: Spacing.sm) { + if let url = LegalLinks.privacyPolicyURL { + Link(destination: url) { + HStack { + Text("settings.privacy.policy") + .font(TypeStyle.body) + .foregroundStyle(palette.accent) + Spacer() + Image(systemName: "arrow.up.right") + .font(.system(size: 12, weight: .semibold)) + .foregroundStyle(palette.textTertiary) + } + } + } + VStack(alignment: .leading, spacing: 4) { + Text("settings.privacy.fullAccess.title") + .font(TypeStyle.caption) + .foregroundStyle(palette.textPrimary) + Text("settings.privacy.fullAccess.body") + .font(TypeStyle.caption2) + .foregroundStyle(palette.textSecondary) + .fixedSize(horizontal: false, vertical: true) + } + } + .cardSurface() + } + } + // MARK: - Reset private var resetButton: some View { diff --git a/OSGKeyboard/en.lproj/Localizable.strings b/OSGKeyboard/en.lproj/Localizable.strings index 7ea665c..cf532fd 100644 --- a/OSGKeyboard/en.lproj/Localizable.strings +++ b/OSGKeyboard/en.lproj/Localizable.strings @@ -3,7 +3,21 @@ "OSGKeyboard" = "OSGKeyboard"; /* Onboarding */ -"onboarding.welcome.subtitle" = "Hold to talk. Release for polished text, in any app."; +"onboarding.welcome.subtitle" = "Tap to talk. Tap again to finish — polished text in any app."; +"onboarding.enable.fullAccessNote" = "Full Access lets the keyboard reach the microphone and your LLM API. We never log what you type."; +"onboarding.permission.mic.title" = "Microphone access"; +"onboarding.permission.mic.body" = "Required for voice input. Audio is processed on your device for transcription."; +"onboarding.permission.mic.allow" = "Allow microphone"; +"onboarding.permission.mic.deniedHint" = "Microphone was denied. Open Settings to enable it, or tap Next to continue."; +"onboarding.permission.speech.title" = "Speech recognition"; +"onboarding.permission.speech.body" = "On-device speech recognition turns your voice into text. Audio never leaves your device."; +"onboarding.permission.speech.allow" = "Allow speech recognition"; +"onboarding.permission.speech.deniedHint" = "Speech recognition was denied. Open Settings to enable it, or tap Next to continue."; +"onboarding.permission.openSettings" = "Open Settings"; +"onboarding.permission.status.undetermined" = "Not requested yet"; +"onboarding.permission.status.granted" = "Allowed"; +"onboarding.permission.status.denied" = "Denied"; +"legal.privacyPolicy" = "Privacy Policy"; "onboarding.enable.title" = "Enable OSGKeyboard"; "onboarding.enable.step1" = "Settings → General → Keyboard → Keyboards"; "onboarding.enable.step2" = "Tap “Add New Keyboard…” and choose OSGKeyboard"; @@ -95,6 +109,10 @@ "settings.onDeviceOnly.body" = "Disable cloud fallback. The keyboard reports an error instead of going online."; "settings.legend.onDevice" = "On-device"; "settings.legend.cloudFallback" = "Cloud fallback"; +"settings.privacy.title" = "Privacy"; +"settings.privacy.policy" = "Privacy Policy"; +"settings.privacy.fullAccess.title" = "About Full Access"; +"settings.privacy.fullAccess.body" = "Full Access is required for the microphone and to read your API key. OSGKeyboard does not record or upload what you type with the keyboard."; /* App group error */ "appGroup.error.title" = "App Group not configured"; @@ -105,8 +123,9 @@ /* Keyboard preview */ "preview.title" = "Keyboard Preview"; -"preview.subtitle" = "Tap the disc to start/stop recording. The real keyboard uses the same layout."; -"preview.placeholder" = "Type or tap to record"; +"preview.subtitle" = "Tap Start/Stop Recording to test here. On the real keyboard, tap the mic disc to dictate."; +"preview.placeholder" = "Type here or tap the record button"; +"preview.placeholder" = "Type here or tap the record button"; "preview.clear" = "Clear text"; "preview.openSettingsA11y" = "Open OSGKeyboard settings"; "preview.modeChip.cycle" = "Cycle input mode"; @@ -118,7 +137,7 @@ "preview.error.engineStart" = "Engine start failed: %@"; /* Keyboard (ext) */ -"keyboard.placeholder.idle" = "Hold to talk"; +"keyboard.placeholder.idle" = "Tap to talk"; "keyboard.placeholder.preparing" = "Preparing"; "keyboard.placeholder.processing" = "Processing"; "keyboard.placeholder.error" = "Polishing failed"; @@ -129,7 +148,7 @@ "keyboard.denied.speech" = "Speech denied"; "keyboard.openSettingsA11y" = "Open OSGKeyboard settings"; "keyboard.deniedHint" = "Opens the OSGKeyboard settings page where you can grant microphone or speech recognition access."; -"keyboard.pressToTalkA11y" = "Push to talk"; +"keyboard.tapToTalkA11y" = "Tap to talk"; /* Mode chip labels (used in both ext + preview stub) */ "mode.off" = "Off"; @@ -146,3 +165,21 @@ "locale.en-US" = "English (US)"; "locale.ja-JP" = "Japanese"; "locale.ko-KR" = "Korean"; + +/* Flow session */ +"flow.error.noSpeech" = "No speech detected. Please try again."; +"keyboard.denied.mic" = "Microphone access denied"; +"keyboard.denied.speech" = "Speech recognition denied"; + +/* Home · Flow session */ +"home.flow.active" = "Voice session active"; +"home.flow.inactive" = "Voice session inactive"; +"home.flow.hint" = "Voice session starts automatically. Switch to any app and tap the keyboard mic to dictate."; +"home.flow.starting" = "Starting voice session…"; +"home.flow.openSettings" = "Open Settings to grant permissions"; +"home.flow.start" = "Start voice session"; +"home.flow.end" = "End voice session"; +"flow.error.speechRequired" = "Speech recognition access is required for voice sessions."; +"flow.error.micRequired" = "Microphone access is required for background voice sessions."; +"flow.error.micUnavailable" = "Microphone is unavailable on this device."; +"flow.error.audioUnavailable" = "Could not start background audio."; diff --git a/OSGKeyboard/zh-Hans.lproj/Localizable.strings b/OSGKeyboard/zh-Hans.lproj/Localizable.strings index 02bf4b6..b7efb02 100644 --- a/OSGKeyboard/zh-Hans.lproj/Localizable.strings +++ b/OSGKeyboard/zh-Hans.lproj/Localizable.strings @@ -3,7 +3,21 @@ "OSGKeyboard" = "OSGKeyboard"; /* Onboarding */ -"onboarding.welcome.subtitle" = "按住说话,松开即得润色文字。"; +"onboarding.welcome.subtitle" = "点按说话，再点结束 — 任意 App 里都能获得润色文字。"; +"onboarding.enable.fullAccessNote" = "完全访问用于麦克风与 API 配置读取。我们不会记录或上传你的击键内容。"; +"onboarding.permission.mic.title" = "麦克风权限"; +"onboarding.permission.mic.body" = "语音输入需要麦克风。音频在设备端转录，不会上传原始录音。"; +"onboarding.permission.mic.allow" = "允许麦克风"; +"onboarding.permission.mic.deniedHint" = "麦克风被拒绝。可前往设置开启，或点「下一步」继续。"; +"onboarding.permission.speech.title" = "语音识别权限"; +"onboarding.permission.speech.body" = "端侧语音识别将语音转为文字。音频不会离开你的设备。"; +"onboarding.permission.speech.allow" = "允许语音识别"; +"onboarding.permission.speech.deniedHint" = "语音识别被拒绝。可前往设置开启，或点「下一步」继续。"; +"onboarding.permission.openSettings" = "打开设置"; +"onboarding.permission.status.undetermined" = "尚未请求"; +"onboarding.permission.status.granted" = "已允许"; +"onboarding.permission.status.denied" = "已拒绝"; +"legal.privacyPolicy" = "隐私政策"; "onboarding.enable.title" = "启用 OSGKeyboard"; "onboarding.enable.step1" = "设置 → 通用 → 键盘 → 键盘"; "onboarding.enable.step2" = "点击「添加新键盘…」并选择 OSGKeyboard"; @@ -95,6 +109,10 @@ "settings.onDeviceOnly.body" = "禁用云端回退,识别失败时会报错而非联网。"; "settings.legend.onDevice" = "端侧识别"; "settings.legend.cloudFallback" = "需联网"; +"settings.privacy.title" = "隐私"; +"settings.privacy.policy" = "隐私政策"; +"settings.privacy.fullAccess.title" = "关于完全访问"; +"settings.privacy.fullAccess.body" = "完全访问用于麦克风与读取 API Key。OSGKeyboard 不会记录或上传你在键盘上的击键内容。"; /* App group error */ "appGroup.error.title" = "App Group 未配置"; @@ -105,8 +123,8 @@ /* Keyboard preview */ "preview.title" = "键盘预览"; -"preview.subtitle" = "点按 disc 开始/结束录音;真实键盘使用同样布局。"; -"preview.placeholder" = "试着输入或按 disc 录音"; +"preview.subtitle" = "点击「开始录音」/「停止录音」按钮测试；真实键盘为长按麦克风圆盘。"; +"preview.placeholder" = "试着输入或点击按钮录音"; "preview.clear" = "清空"; "preview.openSettingsA11y" = "打开 OSGKeyboard 设置"; "preview.modeChip.cycle" = "切换输入模式"; @@ -146,3 +164,21 @@ "locale.en-US" = "English (US)"; "locale.ja-JP" = "日本語"; "locale.ko-KR" = "한국어"; + +/* Flow session */ +"flow.error.noSpeech" = "未检测到语音，请重试。"; +"keyboard.denied.mic" = "麦克风权限被拒绝"; +"keyboard.denied.speech" = "语音识别权限被拒绝"; + +/* Home · Flow session */ +"home.flow.active" = "语音会话进行中"; +"home.flow.inactive" = "语音会话未启动"; +"home.flow.hint" = "语音会话会自动启动。切到任意 App，点按键盘麦克风即可说话。"; +"home.flow.starting" = "正在启动语音会话…"; +"home.flow.openSettings" = "前往设置授予权限"; +"home.flow.start" = "启动语音会话"; +"home.flow.end" = "结束语音会话"; +"flow.error.speechRequired" = "需要语音识别权限才能使用语音会话。"; +"flow.error.micRequired" = "需要麦克风权限才能维持后台语音会话。"; +"flow.error.micUnavailable" = "当前设备无法使用麦克风。"; +"flow.error.audioUnavailable" = "无法启动后台音频。"; diff --git a/OSGKeyboardExt/Info.plist b/OSGKeyboardExt/Info.plist index 0374298..11f5b29 100644 --- a/OSGKeyboardExt/Info.plist +++ b/OSGKeyboardExt/Info.plist @@ -39,8 +39,8 @@ $(PRODUCT_MODULE_NAME).KeyboardViewController NSMicrophoneUsageDescription - OSGKeyboard needs microphone access to transcribe your voice into text. + OSGKeyboard uses the microphone for voice dictation and keeps a background audio session active while a voice session is running. NSSpeechRecognitionUsageDescription - OSGKeyboard uses on-device speech recognition to transcribe your dictation. Audio never leaves your device. + OSGKeyboard uses on-device speech recognition to transcribe your voice. Audio is processed on your device and is not uploaded for transcription. diff --git a/OSGKeyboardExt/KeyboardViewController.swift b/OSGKeyboardExt/KeyboardViewController.swift index b5c58ec..01a187c 100644 --- a/OSGKeyboardExt/KeyboardViewController.swift +++ b/OSGKeyboardExt/KeyboardViewController.swift @@ -4,7 +4,7 @@ // Principal class for the Custom Keyboard Extension. Hosts a single // SwiftUI tree (`KeyboardRootView`) and drives the recording pipeline: // -// AudioCaptureService ──► ASRService ──► PolishingService ──► insertText +// host app dictation handoff ──► App Group transcript ──► insertText // // Design notes: // • The class is `@MainActor` — every UI mutation and `textDocumentProxy` @@ -19,12 +19,22 @@ import UIKit import SwiftUI -import AVFoundation import OSGKeyboardShared @objc(KeyboardViewController) @MainActor public final class KeyboardViewController: UIInputViewController { + private enum FlowWatchdog { + static let pollIntervalNs: UInt64 = 200_000_000 + /// Give the user time to manually open the host app when auto-jump fails. + static let startTimeout: TimeInterval = 30 + static let resultTimeout: TimeInterval = 45 + } + + private enum DictationWatchdog { + static let pollIntervalNs: UInt64 = 400_000_000 + static let timeout: TimeInterval = 45 + } // MARK: - View model @@ -37,17 +47,21 @@ public final class KeyboardViewController: UIInputViewController { // MARK: - State private let state = State() - private let audio = AudioCaptureService() - private let asr: ASRService = ASRServiceFactory.make() private let polisher = PolishingService() - private let permissions = PermissionManager() private let persistor = AppGroupPersistor() - private var session: AudioCaptureService.Session? - private var asrTask: Task? - private var levelTask: Task? - private var hosting: UIHostingController! + /// Legacy one-shot handoff (`osgkeyboard://dictate`). + private var awaitingDictationResult = false + private var dictationRequestStartedAt: TimeInterval = 0 + private var dictationWatchdogTask: Task? + /// Flow session: waiting for host app to come alive after `startflow`. + private var isPendingFlowStart = false + private var flowStartDeadline: TimeInterval = 0 + private var isFlowRecording = false + private var flowWatchdogTask: Task? + private var utteranceTimerTask: Task? + private var utteranceStartedAt: TimeInterval = 0 // MARK: - Lifecycle @@ -60,6 +74,8 @@ public final class KeyboardViewController: UIInputViewController { installStateActions() installSwiftUI() loadPersistedConfig() + consumePendingDictationResultIfNeeded() + refreshDictationProgressStateIfNeeded() } public override func viewWillDisappear(_ animated: Bool) { @@ -67,6 +83,12 @@ public final class KeyboardViewController: UIInputViewController { cancelPipeline() } + public override func viewWillAppear(_ animated: Bool) { + super.viewWillAppear(animated) + consumePendingDictationResultIfNeeded() + refreshDictationProgressStateIfNeeded() + } + public override func didReceiveMemoryWarning() { super.didReceiveMemoryWarning() cancelPipeline() @@ -74,7 +96,8 @@ public final class KeyboardViewController: UIInputViewController { public override func textDidChange(_ textInput: (any UITextInput)?) { super.textDidChange(textInput) - // Hook for future per-app mode switching (e.g. password field → .off). + consumePendingDictationResultIfNeeded() + refreshDictationProgressStateIfNeeded() } // MARK: - Wiring @@ -82,7 +105,7 @@ public final class KeyboardViewController: UIInputViewController { private func installStateActions() { state.beginRecording = { [weak self] in self?.pressBegan() } state.endRecording = { [weak self] in self?.pressEnded() } - state.tapMic = { [weak self] in self?.advanceToNextInputMode() } + state.tapMic = { [weak self] in self?.toggleRecording() } state.openSettings = { [weak self] in self?.openHostApp() } state.setMode = { [weak self] m in self?.persistMode(m) } state.setLocale = { [weak self] l in self?.persistLocale(l) } @@ -125,11 +148,18 @@ public final class KeyboardViewController: UIInputViewController { // MARK: - Press handlers + private func toggleRecording() { + switch state.phase { + case .recording: + pressEnded() + case .idle, .denied, .error: + pressBegan() + case .requestingPermissions, .processing: + break + } + } + private func pressBegan() { - // Allow re-entry from `.denied` and from a finished/cleared - // `.error` so the user can simply press the mic again after - // returning from Settings with permission granted — they - // shouldn't have to wait for an auto-clear timer. switch state.phase { case .idle, .denied, .error: break @@ -137,106 +167,213 @@ public final class KeyboardViewController: UIInputViewController { return } guard state.mode != .off else { return } - // Set the intermediate phase SYNCHRONOUSLY so a rapid second - // press (before the first Task has had a chance to flip phase to - // .recording) is rejected by the guard above. This fixes the race - // where the user double-tapped the mic and we started two - // pipelines at once. - state.phase = .requestingPermissions - Task { @MainActor [weak self] in - guard let self else { return } - let micGranted = await self.permissions.requestMicPermission() - guard micGranted else { - self.state.phase = .denied(.mic) - return - } - // We explicitly ask for Speech recognition permission here. - // `SpeechAnalyzer` does not expose a dedicated request API, - // so the app still relies on the shared Speech permission - // gate and `NSSpeechRecognitionUsageDescription`. - let speechGranted = await self.permissions.requestSpeechPermission() - guard speechGranted else { - self.state.phase = .denied(.speech) - return - } - self.startPipeline() + guard hasFullAccess else { + let msg = "请在系统设置中为 OSGKeyboard 开启“允许完全访问”，否则无法使用语音输入" + state.phase = .error(.unknown(msg), message: msg) + scheduleAutoClearError() + return + } + guard AppGroup.isAvailable else { + let msg = "App Group 未配置，键盘无法与主 App 通信。请重新安装并检查签名配置。" + state.phase = .error(.appGroupUnavailable, message: msg) + scheduleAutoClearError() + return + } + + if FlowSessionBridge.isSessionActive() { + startFlowRecording() + } else { + beginFlowStart() } } private func pressEnded() { - guard state.phase == .recording else { return } - stopPipeline() + if isPendingFlowStart { + cancelPendingFlowStart() + return + } + guard isFlowRecording else { return } + + isFlowRecording = false + stopUtteranceCountdown() + FlowSessionBridge.setRecordingState(.stopped) + state.phase = .processing + state.lastTranscript = "识别中..." + startFlowResultWatchdog() } - // MARK: - Pipeline + private func startFlowRecording() { + isPendingFlowStart = false + flowStartDeadline = 0 + stopFlowWatchdog() - private func startPipeline() { - let session = audio.start() - self.session = session - state.phase = .recording - state.level = 0 + FlowSessionBridge.setTranscriptionLanguage(state.localeId) + FlowSessionBridge.setRecordingState(.recording) + isFlowRecording = true state.lastTranscript = "" + state.phase = .recording + startUtteranceCountdown() + startFlowLevelWatchdog() + debug("startFlowRecording") + } - let locale = resolveLocale(state.localeId) - let events = asr.transcribe( - stream: session.audio, - locale: locale - ) - - asrTask = Task { @MainActor [weak self] in - guard let self else { return } - var lastPartial: String = "" - for await event in events { - switch event { - case .capability(let onDevice): - self.state.onDeviceSupported = onDevice - case .partial(let s): - lastPartial = s - self.state.lastTranscript = s - case .final(let s): - let transcript = s.isEmpty ? lastPartial : s - self.handleFinalTranscript(transcript) - case .error(let m): - self.state.phase = .error(.asr(m)) - self.scheduleAutoClearError() + private func startUtteranceCountdown() { + utteranceStartedAt = Date().timeIntervalSince1970 + state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration) + utteranceTimerTask?.cancel() + utteranceTimerTask = Task { @MainActor [weak self] in + while let self, self.isFlowRecording, !Task.isCancelled { + let elapsed = Date().timeIntervalSince1970 - self.utteranceStartedAt + let remaining = max(0, Int(ceil(FlowSessionKeys.maxUtteranceDuration - elapsed))) + self.state.utteranceRemainingSeconds = remaining + if remaining <= 0 { + self.pressEnded() + return } - } - } - - levelTask = Task { @MainActor [weak self] in - for await level in session.levels { - guard let self else { return } - // Smooth a little extra to feel natural. - self.state.level = Double(self.state.level) * 0.6 + Double(level.meter) * 0.4 + try? await Task.sleep(nanoseconds: 200_000_000) } } } - private func stopPipeline() { - session?.stop() - session = nil - asrTask?.cancel(); asrTask = nil - levelTask?.cancel(); levelTask = nil + private func stopUtteranceCountdown() { + utteranceTimerTask?.cancel() + utteranceTimerTask = nil + state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration) + } + + private func beginFlowStart() { + isPendingFlowStart = true + isFlowRecording = false + flowStartDeadline = Date().timeIntervalSince1970 + FlowWatchdog.startTimeout + state.lastTranscript = "正在启动语音会话..." + state.phase = .processing + openHostApp(path: "startflow") + startFlowStartWatchdog() + debug("beginFlowStart") + } + + private func cancelPendingFlowStart() { + isPendingFlowStart = false + flowStartDeadline = 0 + stopFlowWatchdog() + state.phase = .idle + state.lastTranscript = "" + } + + private func startFlowStartWatchdog() { + stopFlowWatchdog() + flowWatchdogTask = Task { @MainActor [weak self] in + while let self, !Task.isCancelled, self.isPendingFlowStart { + if FlowSessionBridge.isSessionActive() { + self.startFlowRecording() + return + } + let now = Date().timeIntervalSince1970 + if self.flowStartDeadline > 0, now > self.flowStartDeadline { + self.isPendingFlowStart = false + self.flowStartDeadline = 0 + self.showManualSettingsHint(path: "startflow") + return + } + try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs) + } + } + } + + private func startFlowLevelWatchdog() { + stopFlowWatchdog() + flowWatchdogTask = Task { @MainActor [weak self] in + while let self, !Task.isCancelled, self.isFlowRecording { + let levels = FlowSessionBridge.audioLevels() + if let peak = levels.max(), peak > 0 { + self.state.level = Double(peak) + } + try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs) + } + } + } + + private func startFlowResultWatchdog() { + stopFlowWatchdog() + let startedAt = Date().timeIntervalSince1970 + flowWatchdogTask = Task { @MainActor [weak self] in + while let self, !Task.isCancelled { + if let result = FlowSessionBridge.consumeTranscriptionResult() { + self.stopFlowWatchdog() + self.handleFlowTranscript(result) + return + } + if let error = FlowSessionBridge.consumeTranscriptionError() { + self.stopFlowWatchdog() + self.state.phase = .error(.unknown(error), message: error) + self.scheduleAutoClearError() + return + } + let now = Date().timeIntervalSince1970 + if now - startedAt > FlowWatchdog.resultTimeout { + self.stopFlowWatchdog() + let msg = "等待识别结果超时，请重试" + self.state.phase = .error(.unknown(msg), message: msg) + self.scheduleAutoClearError() + return + } + try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs) + } + } + } + + private func handleFlowTranscript(_ transcript: String) { + let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { + state.phase = .idle + state.level = 0 + return + } + // Host app already polished when configured; keyboard only inserts. + textDocumentProxy.insertText(trimmed) + state.lastTranscript = "" + state.level = 0 + state.phase = .idle + debug("flow insert length=\(trimmed.count)") + } + + private func stopFlowWatchdog() { + flowWatchdogTask?.cancel() + flowWatchdogTask = nil } private func cancelPipeline() { - stopPipeline() - asr.cancel() - if state.phase == .recording || state.phase == .processing { + if isFlowRecording || isPendingFlowStart { + if isFlowRecording { + FlowSessionBridge.setRecordingState(.aborted) + } + isFlowRecording = false + isPendingFlowStart = false + stopUtteranceCountdown() + stopFlowWatchdog() + state.level = 0 + } + if awaitingDictationResult { + debug("cancelPipeline ignored while awaiting legacy handoff result") + return + } + if state.phase == .processing { state.phase = .idle } - state.level = 0 - // Reset the on-device flag so the StatusBadge stops showing the - // cloud-fallback indicator between recordings. - state.onDeviceSupported = false } private func handleFinalTranscript(_ transcript: String) { let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { + debug("received empty transcript") + awaitingDictationResult = false + stopDictationWatchdog() state.phase = .idle return } + debug("received transcript length=\(trimmed.count)") + awaitingDictationResult = false + stopDictationWatchdog() // Local engine or transcribe mode: insert directly, no LLM call. if state.isLocalEngine || state.mode == .transcribe { textDocumentProxy.insertText(trimmed) @@ -307,18 +444,8 @@ public final class KeyboardViewController: UIInputViewController { persistor.persist(mode: m) if isRecording { if m == .off { - // Switching to .off while recording: drop the partial - // (no insertion, no LLM). User has explicitly disabled - // the keyboard, so we honour that immediately. - stopPipeline() state.phase = .idle state.lastTranscript = "" - } else if m == .transcribe { - // Switching to .transcribe while in .polish: end the - // recording, the partial will flow through - // handleFinalTranscript which inserts the raw text in - // .transcribe mode (no LLM call). - pressEnded() } } } @@ -335,34 +462,123 @@ public final class KeyboardViewController: UIInputViewController { // MARK: - Open host app - private func openHostApp() { - let urlString = "osgkeyboard://settings" - if let url = URL(string: urlString) { - var responder: UIResponder? = self - while let r = responder { - if let app = r as? UIApplication { - app.open(url) - return - } - responder = r.next - } + private func openHostApp(path: String = "settings") { + guard hasFullAccess else { + let msg = "未开启“允许完全访问”，请先在键盘设置中打开" + state.phase = .error(.unknown(msg), message: msg) + scheduleAutoClearError() + return } - if let url = URL(string: UIApplication.openSettingsURLString) { - var responder: UIResponder? = self - while let r = responder { - if let app = r as? UIApplication { - app.open(url); return - } - responder = r.next + guard let url = URL(string: "osgkeyboard://\(path)") else { + handleHostAppOpenResult(path: path, success: false) + return + } + HostAppLauncher.open(url: url, from: self) { [weak self] success in + self?.handleHostAppOpenResult(path: path, success: success) + } + } + + private func handleHostAppOpenResult(path: String, success: Bool) { + debug("openHostApp path=\(path) success=\(success)") + guard !success else { return } + + // Flow start: auto-jump often fails in WeChat/Safari — keep polling + // so a manually opened host app can still satisfy the session check. + if path == "startflow", isPendingFlowStart { + state.lastTranscript = "无法自动跳转，请从主屏幕打开 OSGKeyboard，然后返回继续" + return + } + + if path == "dictate" { + awaitingDictationResult = false + stopDictationWatchdog() + } + showManualSettingsHint(path: path) + } + + private func consumePendingDictationResultIfNeeded() { + guard let transcript = DictationBridge.consumePendingTranscript() else { return } + debug("consumePendingDictationResultIfNeeded success") + handleFinalTranscript(transcript) + } + + private func refreshDictationProgressStateIfNeeded() { + guard awaitingDictationResult, case .processing = state.phase else { return } + let progress = DictationBridge.currentStatus() + switch progress.status { + case .requested: + state.lastTranscript = state.isLocalEngine + ? "正在打开 OSGKeyboard（本地转写）..." + : "正在打开 OSGKeyboard..." + case .recording: + state.lastTranscript = state.isLocalEngine + ? "正在本地录音，请完成后返回当前输入页" + : "正在录音，请完成后返回当前输入页" + case .transcribing: + state.lastTranscript = state.isLocalEngine + ? "本地识别中，请稍候并返回输入页" + : "识别中，请稍候并返回输入页" + case .error: + let msg = progress.message ?? "录音失败，请重试" + debug("host returned error: \(msg)") + awaitingDictationResult = false + stopDictationWatchdog() + state.phase = .error(.unknown(msg), message: msg) + scheduleAutoClearError() + case .cancelled: + debug("host cancelled") + awaitingDictationResult = false + stopDictationWatchdog() + state.phase = .idle + case .done, .idle: + break + } + // Host app can be killed or leave without callback. If status does not + // advance for too long, fail fast with an actionable retry message. + let now = Date().timeIntervalSince1970 + let lastProgressAt = progress.updatedAt > 0 ? progress.updatedAt : dictationRequestStartedAt + if now - lastProgressAt > DictationWatchdog.timeout { + let timeoutMessage = "等待录音结果超时，请返回 OSGKeyboard 完成录音后重试" + debug("dictation timeout after \(Int(now - lastProgressAt))s") + awaitingDictationResult = false + stopDictationWatchdog() + DictationBridge.clear() + state.phase = .error(.unknown(timeoutMessage), message: timeoutMessage) + scheduleAutoClearError() + } + } + + private func showManualSettingsHint(path: String = "settings") { + let msg: String + if !hasFullAccess { + msg = "请先开启 OSGKeyboard 的“允许完全访问”，否则键盘无法跳转到 App" + } else if path == "settings" { + msg = "系统拒绝了键盘跳转。请手动打开 OSGKeyboard App 进入设置页" + } else if path == "startflow" { + msg = "语音会话未启动。请从主屏幕打开 OSGKeyboard App，返回后再按麦克风" + } else if state.isLocalEngine { + msg = "系统拒绝了键盘跳转。请先手动打开 OSGKeyboard 完成本地转写，再返回输入页" + } else { + msg = "系统拒绝了键盘跳转。请先手动打开 OSGKeyboard 录音，再返回输入页" + } + state.phase = .error(.unknown(msg), message: msg) + scheduleAutoClearError() + } + + private func startDictationWatchdog() { + stopDictationWatchdog() + dictationWatchdogTask = Task { @MainActor [weak self] in + while let self, !Task.isCancelled, self.awaitingDictationResult { + self.consumePendingDictationResultIfNeeded() + self.refreshDictationProgressStateIfNeeded() + try? await Task.sleep(nanoseconds: DictationWatchdog.pollIntervalNs) } } } - // MARK: - Helpers - - private func resolveLocale(_ id: String) -> Locale { - if id == "auto" { return .current } - return Locale(identifier: id) + private func stopDictationWatchdog() { + dictationWatchdogTask?.cancel() + dictationWatchdogTask = nil } private func scheduleAutoClearError() { @@ -381,4 +597,10 @@ public final class KeyboardViewController: UIInputViewController { } } } + + private func debug(_ message: String) { + #if DEBUG + print("🎙️[KeyboardVC] \(message)") + #endif + } } diff --git a/OSGKeyboardExt/OSGKeyboardExt.entitlements b/OSGKeyboardExt/OSGKeyboardExt.entitlements index 2febc2b..af99559 100644 --- a/OSGKeyboardExt/OSGKeyboardExt.entitlements +++ b/OSGKeyboardExt/OSGKeyboardExt.entitlements @@ -6,9 +6,9 @@ group.com.osgkeyboard.shared - com.apple.security.keychain-access-groups + keychain-access-groups - com.osgkeyboard.shared + $(AppIdentifierPrefix)com.osgkeyboard.shared diff --git a/OSGKeyboardExt/Services/AppGroupPersistor.swift b/OSGKeyboardExt/Services/AppGroupPersistor.swift index 0f53e94..59ef98f 100644 --- a/OSGKeyboardExt/Services/AppGroupPersistor.swift +++ b/OSGKeyboardExt/Services/AppGroupPersistor.swift @@ -62,16 +62,19 @@ public struct AppGroupPersistor { /// Persist `mode` to the App Group store. public func persist(mode: KeyboardViewController.State.InputMode) { + guard AppGroup.isAvailable else { return } AppGroupStore().setModeId(mode.rawValue) } /// Persist `localeId` to the App Group store. public func persist(localeId: String) { + guard AppGroup.isAvailable else { return } AppGroupStore().setLocaleId(localeId) } /// Persist `engineMode` to the App Group store. public func persist(engineMode: String) { + guard AppGroup.isAvailable else { return } AppGroupStore().setEngineMode(engineMode) } } \ No newline at end of file diff --git a/OSGKeyboardExt/Services/HostAppLauncher.swift b/OSGKeyboardExt/Services/HostAppLauncher.swift new file mode 100644 index 0000000..67274a4 --- /dev/null +++ b/OSGKeyboardExt/Services/HostAppLauncher.swift @@ -0,0 +1,70 @@ +// HostAppLauncher.swift +// OSGKeyboard · Keyboard Extension +// +// Opens the host app via URL using every extension-safe strategy: +// 1. `extensionContext.open` (official) +// 2. Responder-chain `UIApplication.open` (TypeWhisper pattern) +// 3. `sharedApplication` KVC fallback (common in full-access keyboards) + +import UIKit + +enum HostAppLauncher { + @MainActor + static func open( + url: URL, + from controller: KeyboardViewController, + completion: @escaping @MainActor (Bool) -> Void + ) { + if let context = controller.extensionContext { + context.open(url) { success in + Task { @MainActor in + if success { + completion(true) + return + } + completion(openViaFallback(url, from: controller)) + } + } + return + } + completion(openViaFallback(url, from: controller)) + } + + @MainActor + private static func openViaFallback( + _ url: URL, + from controller: KeyboardViewController + ) -> Bool { + if openViaResponderChain(url, from: controller) { + return true + } + return openViaSharedApplication(url) + } + + @MainActor + private static func openViaResponderChain( + _ url: URL, + from controller: KeyboardViewController + ) -> Bool { + var responder: UIResponder? = controller + while let current = responder { + if let application = current as? UIApplication { + application.open(url, options: [:]) { _ in } + return true + } + responder = current.next + } + return false + } + + @MainActor + private static func openViaSharedApplication(_ url: URL) -> Bool { + guard + let application = UIApplication.value(forKeyPath: "sharedApplication") as? UIApplication + else { + return false + } + application.open(url, options: [:]) { _ in } + return true + } +} diff --git a/OSGKeyboardExt/Services/PermissionManager.swift b/OSGKeyboardExt/Services/PermissionManager.swift index eecb9b4..2514528 100644 --- a/OSGKeyboardExt/Services/PermissionManager.swift +++ b/OSGKeyboardExt/Services/PermissionManager.swift @@ -47,15 +47,34 @@ public final class PermissionManager: @unchecked Sendable { /// method of its own and the framework checks the same TCC /// entry on first use. public func requestSpeechPermission() async -> Bool { + await Self.requestSpeechPermissionNonisolated() + } + + // MARK: - Nonisolated permission bridge + // + // `SFSpeechRecognizer.requestAuthorization` callback is not guaranteed + // to run on main queue. Building the callback inline inside a + // `@MainActor` method can trigger runtime actor/isolation assertions. + // Keep the continuation + callback creation in nonisolated helpers. + private nonisolated static func requestSpeechPermissionNonisolated() async -> Bool { await withCheckedContinuation { (cont: CheckedContinuation) in - SFSpeechRecognizer.requestAuthorization { status in - switch status { - case .authorized: cont.resume(returning: true) - case .denied, .restricted, .notDetermined: - cont.resume(returning: false) - @unknown default: - cont.resume(returning: false) - } + SFSpeechRecognizer.requestAuthorization( + makeSpeechAuthHandler(continuation: cont) + ) + } + } + + private nonisolated static func makeSpeechAuthHandler( + continuation: CheckedContinuation + ) -> @Sendable (SFSpeechRecognizerAuthorizationStatus) -> Void { + return { status in + switch status { + case .authorized: + continuation.resume(returning: true) + case .denied, .restricted, .notDetermined: + continuation.resume(returning: false) + @unknown default: + continuation.resume(returning: false) } } } diff --git a/OSGKeyboardExt/Views/KeyboardRootView.swift b/OSGKeyboardExt/Views/KeyboardRootView.swift index fdb5e21..59eb721 100644 --- a/OSGKeyboardExt/Views/KeyboardRootView.swift +++ b/OSGKeyboardExt/Views/KeyboardRootView.swift @@ -22,7 +22,7 @@ import SwiftUI import OSGKeyboardShared public struct KeyboardRootView: View { - @Environment(\.themePalette) private var palette: ThemePalette + @Environment(\.colorScheme) private var colorScheme @ObservedObject var state: State @@ -35,6 +35,10 @@ public struct KeyboardRootView: View { /// it up. static let totalHeight: CGFloat = 280 + private var palette: ThemePalette { + colorScheme == .dark ? Palette.dark : Palette.light + } + public var body: some View { VStack(spacing: 0) { topBar @@ -48,26 +52,12 @@ public struct KeyboardRootView: View { } .padding(.top, 4) .padding(.bottom, 6) - // iOS keyboard extensions always render dark (Apple's default - // for custom keyboards), and we let the system UI chrome show - // through by drawing no background of our own. + // Let the system UI chrome show through by drawing no background + // of our own. .background(Color.clear) .frame(height: Self.totalHeight) - // Top edge: subtle highlight gradient + 0.5pt divider line. - // These give the keyboard a "physical surface" feel and visually - // separate it from the host text field above. We overlay (not - // background) so the underlying color stays clear. - .overlay(alignment: .top) { - VStack(spacing: 0) { - Rectangle() - .fill(LinearGradient( - colors: [Color.white.opacity(0.05), .clear], - startPoint: .top, endPoint: .bottom - )) - .frame(height: 1) - Spacer(minLength: 0) - } - } + // Feed the resolved palette to all nested chips/buttons. + .environment(\.themePalette, palette) } // MARK: - Top bar @@ -96,7 +86,7 @@ public struct KeyboardRootView: View { .overlay(Circle().stroke(palette.divider, lineWidth: 0.5)) } .buttonStyle(.plain) - .accessibilityLabel(Text("home.action.openSettingsA11y")) + .accessibilityLabel(Text(KeyboardL10n.openSettingsA11y)) } .padding(.horizontal, Spacing.md) } @@ -115,9 +105,8 @@ public struct KeyboardRootView: View { RecordButton( phase: buttonPhase, level: state.level, - onPressBegan: state.beginRecording, - onPressEnded: state.endRecording, - onTap: state.tapMic + remainingSeconds: state.phase == .recording ? state.utteranceRemainingSeconds : nil, + onToggle: state.tapMic ) .frame(width: 140, height: 140) } @@ -139,7 +128,7 @@ public struct KeyboardRootView: View { state.deleteBackward() } Button(action: state.insertSpace) { - Text("common.space") + Text(KeyboardL10n.space) .font(TypeStyle.body) .foregroundStyle(palette.textPrimary) .frame(maxWidth: .infinity, minHeight: 42) @@ -150,7 +139,7 @@ public struct KeyboardRootView: View { ) } .buttonStyle(.plain) - .accessibilityLabel(Text("common.space")) + .accessibilityLabel(Text(KeyboardL10n.space)) ToolbarIconButton(systemName: "return", label: "newline") { state.insertNewline() } @@ -211,13 +200,13 @@ private struct TranscriptLine: View { ZStack { switch phase { case .idle: - Text("keyboard.placeholder.idle") + Text(KeyboardL10n.placeholderIdle) .font(TypeStyle.caption) .foregroundStyle(palette.textTertiary) case .requestingPermissions: HStack(spacing: 6) { ProgressView().controlSize(.mini).tint(palette.textSecondary) - Text("keyboard.placeholder.preparing") + Text(KeyboardL10n.placeholderPreparing) .font(TypeStyle.caption) .foregroundStyle(palette.textSecondary) } @@ -231,9 +220,11 @@ private struct TranscriptLine: View { case .processing: HStack(spacing: 6) { ProgressView().controlSize(.mini).tint(palette.accent) - Text("keyboard.placeholder.processing") + Text(transcript.isEmpty ? KeyboardL10n.placeholderProcessing : transcript) .font(TypeStyle.caption) .foregroundStyle(palette.textSecondary) + .lineLimit(1) + .truncationMode(.tail) } case .error(_, let msg): Text(msg ?? "") @@ -256,7 +247,7 @@ private struct TranscriptLine: View { .contentShape(Rectangle()) } .buttonStyle(.plain) - .accessibilityHint(Text("keyboard.deniedHint")) + .accessibilityHint(Text(KeyboardL10n.deniedHint)) } } .frame(maxWidth: .infinity) @@ -265,8 +256,8 @@ private struct TranscriptLine: View { private func deniedMessage(for reason: KeyboardViewController.State.Phase.Reason) -> String { switch reason { - case .mic: return "麦克风被拒绝 · Mic denied" - case .speech: return "语音识别被拒绝 · Speech denied" + case .mic: return KeyboardL10n.micDenied + case .speech: return KeyboardL10n.speechDenied } } } @@ -361,7 +352,7 @@ private struct LocalEngineChip: View { var body: some View { HStack(spacing: 4) { Image(systemName: "iphone.badge.checkmark") - Text("keyboard.placeholder.localBadge") + Text(KeyboardL10n.localBadge) } .font(TypeStyle.caption2) .foregroundStyle(palette.accent) @@ -372,6 +363,34 @@ private struct LocalEngineChip: View { } } +// MARK: - Extension text fallback +// +// Custom keyboard extensions can end up without the expected localized +// resource table when signing/project generation drifts. Keep a tiny +// in-code fallback map so UI never shows raw key names like +// "common.space" in production. +private enum KeyboardL10n { + private static var isChinese: Bool { + Locale.preferredLanguages.first?.hasPrefix("zh") == true + } + + static var space: String { isChinese ? "空格" : "Space" } + static var placeholderIdle: String { isChinese ? "点按说话" : "Tap to talk" } + static var placeholderPreparing: String { isChinese ? "准备中…" : "Preparing" } + static var placeholderProcessing: String { isChinese ? "处理中…" : "Processing" } + static var localBadge: String { isChinese ? "本地" : "On-device" } + static var micDenied: String { isChinese ? "麦克风被拒绝" : "Mic denied" } + static var speechDenied: String { isChinese ? "语音识别被拒绝" : "Speech denied" } + static var deniedHint: String { + isChinese + ? "打开 OSGKeyboard 设置,可授予麦克风 / 语音识别权限。" + : "Open OSGKeyboard settings to grant microphone / speech access." + } + static var openSettingsA11y: String { + isChinese ? "打开 OSGKeyboard 设置" : "Open OSGKeyboard settings" + } +} + // MARK: - Mode chip private struct ModeChip: View { diff --git a/OSGKeyboardExt/Views/RecordButton.swift b/OSGKeyboardExt/Views/RecordButton.swift index 79b50b3..cf453cb 100644 --- a/OSGKeyboardExt/Views/RecordButton.swift +++ b/OSGKeyboardExt/Views/RecordButton.swift @@ -1,10 +1,8 @@ // RecordButton.swift // OSGKeyboard · Keyboard Extension // -// The hero control. 120 pt primary disc with a soft inner gradient, a -// breathing outer ring while recording, and a centred waveform that maps -// directly to the real audio RMS. Idle / recording / processing are three -// distinct visual states — no flicker, no surprise transitions. +// Tap-to-toggle mic: tap once to start, tap again to stop. Shows a +// remaining-time countdown while recording; last 10 seconds turn red. import SwiftUI import OSGKeyboardShared @@ -21,38 +19,38 @@ struct RecordButton: View { let phase: Phase let level: Double // 0...1 - let onPressBegan: () -> Void - let onPressEnded: () -> Void - let onTap: () -> Void + /// Seconds left in the current utterance; shown only while recording. + let remainingSeconds: Int? + let onToggle: () -> Void - @GestureState private var isPressed: Bool = false @State private var breath: Bool = false init( phase: Phase, level: Double, - onPressBegan: @escaping () -> Void, - onPressEnded: @escaping () -> Void, - onTap: @escaping () -> Void + remainingSeconds: Int? = nil, + onToggle: @escaping () -> Void ) { self.phase = phase self.level = level - self.onPressBegan = onPressBegan - self.onPressEnded = onPressEnded - self.onTap = onTap + self.remainingSeconds = remainingSeconds + self.onToggle = onToggle + } + + private var isUrgent: Bool { + guard phase == .recording, let remainingSeconds else { return false } + return remainingSeconds <= 10 } var body: some View { ZStack { - // Outer breathing ring (recording only) Circle() - .stroke(palette.recordRed.opacity(0.35), lineWidth: 2) + .stroke(palette.recordRed.opacity(isUrgent ? 0.55 : 0.35), lineWidth: isUrgent ? 3 : 2) .frame(width: 150, height: 150) .scaleEffect(breath ? 1.18 : 0.95) .opacity(phase == .recording ? 1 : 0) .animation(Motion.breath, value: breath) - // Halo: soft red glow that intensifies with input level Circle() .fill( RadialGradient( @@ -68,7 +66,6 @@ struct RecordButton: View { .animation(Motion.soft, value: phase) .animation(Motion.soft, value: level) - // Secondary outer ring (always present, dimmer when idle) Circle() .stroke( Color.white.opacity(phase == .idle ? 0.08 : 0.12), @@ -76,7 +73,6 @@ struct RecordButton: View { ) .frame(width: 140, height: 140) - // Main disc with gradient + soft inner highlight ZStack { Circle() .fill(discGradient) @@ -84,7 +80,6 @@ struct RecordButton: View { .stroke(Color.white.opacity(0.16), lineWidth: 1) .blendMode(.overlay) - // Centre content — switches by phase Group { switch phase { case .idle: @@ -92,17 +87,19 @@ struct RecordButton: View { .font(.system(size: 38, weight: .medium)) .foregroundStyle(palette.textPrimary) case .recording: - WaveformView(level: level, active: true) - .frame(width: 80, height: 44) - .transition(.opacity) + VStack(spacing: 4) { + if let remainingSeconds { + Text(formatRemaining(remainingSeconds)) + .font(.system(size: 22, weight: .semibold, design: .rounded)) + .foregroundStyle(isUrgent ? .white : palette.textPrimary) + .monospacedDigit() + .contentTransition(.numericText()) + } + WaveformView(level: level, active: true) + .frame(width: 72, height: 32) + } + .transition(.opacity) case .processing: - // Scaled to ~50pt inside a 120pt disc (~42%) — - // same absolute size as the preview stub's - // spinner (2.5x of the default ProgressView), - // just a smaller fraction because the real - // disc is bigger. The user gets the same - // visual weight whether they're looking at - // the in-app preview or the live keyboard. ProgressView() .progressViewStyle(.circular) .tint(palette.textPrimary) @@ -115,63 +112,34 @@ struct RecordButton: View { } } .frame(width: 120, height: 120) - .scaleEffect(isPressed ? 0.94 : 1.0) - .animation(Motion.quick, value: isPressed) .animation(Motion.soft, value: phase) + .animation(Motion.soft, value: remainingSeconds) } .contentShape(Circle()) - // Press-to-talk: act on the FIRST touch-down, not after a 150 ms - // minimum duration. That's what Typeless feels like, and it's what - // makes the keyboard feel responsive. A tap (very short press) is - // interpreted as "toggle" for the secondary action (onTap), not - // "record" — the recording only fires if the press lasts long - // enough to read as intentional. This avoids the previous bug - // where every single tap fired both onPressBegan AND onTap. - .gesture( - LongPressGesture(minimumDuration: 0.18) - .sequenced(before: DragGesture(minimumDistance: 0)) - .updating($isPressed) { value, state, _ in - switch value { - case .second(true, _): state = true - default: state = false - } - } - .onChanged { value in - if case .second(true, _) = value, !pressArmed { - pressArmed = true - onPressBegan() - } - } - .onEnded { _ in - if pressArmed { pressArmed = false; onPressEnded() } - } - ) - .simultaneousGesture( - // Pure tap: only fires when the user lifts before the long-press - // threshold. This becomes the "secondary action" (e.g. cycle - // mode). It is paired with, not conflicting with, the long-press. - TapGesture(count: 1) - .onEnded { - if !pressArmed { onTap() } - } - ) + .onTapGesture { + guard phase != .processing else { return } + onToggle() + } .onAppear { breath = (phase == .recording) } .onChange(of: phase) { _, new in breath = (new == .recording) } - .accessibilityLabel(Text("keyboard.pressToTalkA11y")) + .accessibilityLabel(Text("keyboard.tapToTalkA11y")) } - @State private var pressArmed: Bool = false + private func formatRemaining(_ seconds: Int) -> String { + let m = seconds / 60 + let s = seconds % 60 + return String(format: "%d:%02d", m, s) + } private var discGradient: LinearGradient { switch phase { case .recording: - return LinearGradient( - colors: [palette.recordRed.opacity(0.95), palette.recordRed.opacity(0.75)], - startPoint: .top, - endPoint: .bottom - ) + let colors: [Color] = isUrgent + ? [palette.recordRed, palette.recordRed.opacity(0.85)] + : [palette.recordRed.opacity(0.95), palette.recordRed.opacity(0.75)] + return LinearGradient(colors: colors, startPoint: .top, endPoint: .bottom) case .processing: return LinearGradient( colors: [palette.surfaceElevated, palette.surface], @@ -185,10 +153,6 @@ struct RecordButton: View { endPoint: .bottom ) case .idle: - // Brand green — same hue as `Palette.{dark,light}.accent` - // and the AccentColor asset. The disc is the keyboard's - // primary CTA, and a dark-gray disc looked like an inert - // surface, not an actionable button. return LinearGradient( colors: [ palette.accent.opacity(0.95), diff --git a/OSGKeyboardExt/en.lproj/Localizable.strings b/OSGKeyboardExt/en.lproj/Localizable.strings index 3a7408a..cbce72c 100644 --- a/OSGKeyboardExt/en.lproj/Localizable.strings +++ b/OSGKeyboardExt/en.lproj/Localizable.strings @@ -105,15 +105,15 @@ /* Keyboard preview */ "preview.title" = "Keyboard Preview"; -"preview.subtitle" = "Tap the disc to start/stop recording. The real keyboard uses the same layout."; -"preview.placeholder" = "Type or tap to record"; +"preview.subtitle" = "Tap Start/Stop Recording to test here. On the real keyboard, press and hold the mic disc."; +"preview.placeholder" = "Type here or tap the record button"; "preview.clear" = "Clear text"; "preview.openSettingsA11y" = "Open OSGKeyboard settings"; "preview.modeChip.cycle" = "Cycle input mode"; "preview.localeChip.cycle" = "Cycle recognition language"; /* Keyboard (ext) */ -"keyboard.placeholder.idle" = "Hold to talk"; +"keyboard.placeholder.idle" = "Tap to talk"; "keyboard.placeholder.preparing" = "Preparing"; "keyboard.placeholder.processing" = "Processing"; "keyboard.placeholder.error" = "Polishing failed"; @@ -124,7 +124,7 @@ "keyboard.denied.speech" = "Speech denied"; "keyboard.openSettingsA11y" = "Open OSGKeyboard settings"; "keyboard.deniedHint" = "Opens the OSGKeyboard settings page where you can grant microphone or speech recognition access."; -"keyboard.pressToTalkA11y" = "Push to talk"; +"keyboard.tapToTalkA11y" = "Tap to talk"; /* Mode chip labels (used in both ext + preview stub) */ "mode.off" = "Off"; diff --git a/OSGKeyboardExt/zh-Hans.lproj/Localizable.strings b/OSGKeyboardExt/zh-Hans.lproj/Localizable.strings index cdaced8..fc52c10 100644 --- a/OSGKeyboardExt/zh-Hans.lproj/Localizable.strings +++ b/OSGKeyboardExt/zh-Hans.lproj/Localizable.strings @@ -105,15 +105,15 @@ /* Keyboard preview */ "preview.title" = "键盘预览"; -"preview.subtitle" = "点按 disc 开始/结束录音;真实键盘使用同样布局。"; -"preview.placeholder" = "试着输入或按 disc 录音"; +"preview.subtitle" = "点击「开始录音」/「停止录音」按钮测试；真实键盘为长按麦克风圆盘。"; +"preview.placeholder" = "试着输入或点击按钮录音"; "preview.clear" = "清空"; "preview.openSettingsA11y" = "打开 OSGKeyboard 设置"; "preview.modeChip.cycle" = "切换输入模式"; "preview.localeChip.cycle" = "切换识别语言"; /* Keyboard (ext) */ -"keyboard.placeholder.idle" = "按住说话"; +"keyboard.placeholder.idle" = "点按说话"; "keyboard.placeholder.preparing" = "准备中…"; "keyboard.placeholder.processing" = "处理中…"; "keyboard.placeholder.error" = "润色失败"; @@ -124,7 +124,7 @@ "keyboard.denied.speech" = "语音识别被拒绝"; "keyboard.openSettingsA11y" = "打开 OSGKeyboard 设置"; "keyboard.deniedHint" = "打开 OSGKeyboard 设置,可授予麦克风 / 语音识别权限。"; -"keyboard.pressToTalkA11y" = "按住说话"; +"keyboard.tapToTalkA11y" = "点按说话"; /* Mode chip labels */ "mode.off" = "关闭"; diff --git a/OSGKeyboardShared/Constants/AppGroup.swift b/OSGKeyboardShared/Constants/AppGroup.swift index 50ff542..eeb84e1 100644 --- a/OSGKeyboardShared/Constants/AppGroup.swift +++ b/OSGKeyboardShared/Constants/AppGroup.swift @@ -12,18 +12,15 @@ public enum AppGroup { /// Whether the App Group container is available on this device. /// - /// Cached at first read — the underlying `UserDefaults(suiteName:)` - /// call is cheap, but main-app startup and every keyboard-extension - /// read hit it, so we memoize the result. - /// - /// Production code paths MUST go through `isAvailable` first and - /// surface a friendly error view (e.g. `AppGroupErrorView`) on the - /// main app, or the keyboard extension's persisted-locale load. - /// Calling `defaults` directly when the group is missing will trip - /// the DEBUG `fatalError` below — that path is reserved for - /// developer-only escape hatches and intentional debugging. + /// Checks both `UserDefaults(suiteName:)` *and* the on-disk container. + /// The suite alone can appear to open while the container is still `(null)` + /// when provisioning is misconfigured — that case produces the + /// `CFPrefsPlistSource … Container: (null)` console warning. public static let isAvailable: Bool = { - UserDefaults(suiteName: identifier) != nil + guard UserDefaults(suiteName: identifier) != nil else { return false } + return FileManager.default.containerURL( + forSecurityApplicationGroupIdentifier: identifier + ) != nil }() /// Shared UserDefaults instance for cross-process config. diff --git a/OSGKeyboardShared/Models/EngineServiceLabel.swift b/OSGKeyboardShared/Models/EngineServiceLabel.swift new file mode 100644 index 0000000..bc2cf1f --- /dev/null +++ b/OSGKeyboardShared/Models/EngineServiceLabel.swift @@ -0,0 +1,26 @@ +// EngineServiceLabel.swift +// OSGKeyboard · Shared +// +// Human-readable summary of the active engine / AI provider for UI hints. + +import Foundation + +public enum EngineServiceLabel { + public static func summary( + engineMode: String, + providerId: String, + model: String + ) -> String { + let isChinese = Locale.preferredLanguages.first?.hasPrefix("zh") == true + let prefix = isChinese ? "当前：" : "Active: " + if engineMode == "local" { + return isChinese + ? "\(prefix)本地引擎 · Apple SpeechAnalyzer" + : "\(prefix)On-device · Apple SpeechAnalyzer" + } + let provider = LLMProvider.provider(id: providerId) + let trimmedModel = model.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmedModel.isEmpty { return "\(prefix)\(provider.name)" } + return "\(prefix)\(provider.name) · \(trimmedModel)" + } +} diff --git a/OSGKeyboardShared/Models/ProviderConfig.swift b/OSGKeyboardShared/Models/ProviderConfig.swift index 2776c21..b6c99f0 100644 --- a/OSGKeyboardShared/Models/ProviderConfig.swift +++ b/OSGKeyboardShared/Models/ProviderConfig.swift @@ -27,6 +27,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable { static let modeId = "config.modeId" static let localeId = "config.localeId" static let engineMode = "config.engineMode" + static let hasCompletedOnboarding = "config.hasCompletedOnboarding" } @Published public var providerId: String { @@ -66,6 +67,9 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable { @Published public var engineMode: String { didSet { defaults.set(engineMode, forKey: Key.engineMode) } } + @Published public var hasCompletedOnboarding: Bool { + didSet { defaults.set(hasCompletedOnboarding, forKey: Key.hasCompletedOnboarding) } + } public var isConfigured: Bool { // Local engine (on-device ASR only) doesn't need an API key, @@ -73,10 +77,13 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable { // Treat it as always-configured so onboarding's "Next" button // enables the moment the user picks the local path, instead // of forcing them to fill in cloud fields they won't use. - if engineMode == "local" { return true } + if isLocalEngine { return true } return !baseURL.isEmpty && !apiKey.isEmpty && !model.isEmpty } + /// On-device ASR only — no cloud LLM polish. + public var isLocalEngine: Bool { engineMode == "local" } + /// The system prompt the user *sees* in the editor — fall back to the /// provider-aware default from `AppGroupStore` when nothing is set. public var defaultSystemPrompt: String { @@ -85,25 +92,27 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable { private let defaults: UserDefaults - public init(defaults: UserDefaults = AppGroup.defaults) { - self.defaults = defaults - let pid = defaults.string(forKey: Key.providerId) ?? "openai" + public init(defaults: UserDefaults? = nil) { + let resolvedDefaults: UserDefaults = defaults ?? (AppGroup.isAvailable ? AppGroup.defaults : .standard) + self.defaults = resolvedDefaults + let pid = resolvedDefaults.string(forKey: Key.providerId) ?? "openai" let preset = LLMProvider.provider(id: pid) self.providerId = pid - self.baseURL = defaults.string(forKey: Key.baseURL) ?? preset.defaultBaseURL + self.baseURL = resolvedDefaults.string(forKey: Key.baseURL) ?? preset.defaultBaseURL // Resolve the API key with a one-shot migration from the legacy // UserDefaults slot. After this runs once, `Key.apiKeyLegacy` // is empty in the suite and all subsequent reads go through the // Keychain. - self.apiKey = ProviderConfig.resolveAPIKey(defaults: defaults) + self.apiKey = ProviderConfig.resolveAPIKey(defaults: resolvedDefaults) - self.model = defaults.string(forKey: Key.model) ?? preset.defaultModel - self.systemPrompt = defaults.string(forKey: Key.systemPrompt) + self.model = resolvedDefaults.string(forKey: Key.model) ?? preset.defaultModel + self.systemPrompt = resolvedDefaults.string(forKey: Key.systemPrompt) ?? AppGroupStore.defaultSystemPrompt(for: pid) - self.modeId = defaults.string(forKey: Key.modeId) ?? "polish" - self.localeId = defaults.string(forKey: Key.localeId) ?? "auto" - self.engineMode = defaults.string(forKey: Key.engineMode) ?? "cloud" + self.modeId = resolvedDefaults.string(forKey: Key.modeId) ?? "polish" + self.localeId = resolvedDefaults.string(forKey: Key.localeId) ?? "auto" + self.engineMode = resolvedDefaults.string(forKey: Key.engineMode) ?? "cloud" + self.hasCompletedOnboarding = resolvedDefaults.bool(forKey: Key.hasCompletedOnboarding) } /// Read the API key from the Keychain, falling back to a one-time diff --git a/OSGKeyboardShared/Services/ASRService.swift b/OSGKeyboardShared/Services/ASRService.swift index 003b1b5..b2c6cb4 100644 --- a/OSGKeyboardShared/Services/ASRService.swift +++ b/OSGKeyboardShared/Services/ASRService.swift @@ -114,72 +114,111 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable { private let lock = OSAllocatedUnfairLock() private var analyzer: SpeechAnalyzer? private var analyzerTask: Task? + private var analyzerFinished = false + + /// Canonical capture format: 16 kHz mono Float32 from `AudioCaptureService` + /// / `PreviewASRController` before it reaches SpeechAnalyzer. + private static let captureFormat = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: 16_000, + channels: 1, + interleaved: false + )! func transcribe( stream: AsyncStream, locale: Locale ) -> AsyncStream { AsyncStream { continuation in - // SpeechAnalyzer is always fully on-device. continuation.yield(.capability(onDeviceSupported: true)) - let transcriber = DictationTranscriber(locale: locale, preset: .progressiveShortDictation) - let newAnalyzer = SpeechAnalyzer(modules: [transcriber]) - self.lock.withLock { self.analyzer = newAnalyzer } - - // iOS 26's `DictationTranscriber` requires **Int16** PCM - // (precondition `"Audio sample data must be 16-bit signed - // integers"` — the legacy recognizer used Float32 at this - // boundary; `SpeechAnalyzer` is strict Int16). 16 kHz mono, Int16, - // interleaved — the canonical layout Apple's Speech - // framework examples use. - let audioFormat = AVAudioFormat( - commonFormat: .pcmFormatInt16, - sampleRate: 16_000, - channels: 1, - interleaved: true - )! - let task = Task { [weak self] in guard let self else { return } - do { - try await newAnalyzer.prepareToAnalyze(in: audioFormat) - - let inputStream = self.makeInputStream(from: stream, format: audioFormat) - - // Feed audio in a child task so we can concurrently - // iterate `transcriber.results` on the outer task. - // After the audio stream ends, finalize so the results - // sequence can drain and complete. - let feedTask = Task { - do { - try await newAnalyzer.start(inputSequence: inputStream) - try await newAnalyzer.finalizeAndFinishThroughEndOfInput() - } catch {} + self.lock.withLock { self.analyzerFinished = false } + defer { + self.lock.withLock { + self.analyzer = nil + self.analyzerTask = nil + self.analyzerFinished = true } - defer { feedTask.cancel() } + } - var lastText = "" + do { + guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else { + Self.debug("locale unsupported: \(locale.identifier(.bcp47))") + continuation.yield(.error("当前系统未分配可用语音语言模型，请稍后重试或切换语言")) + continuation.finish() + return + } + let transcriber = DictationTranscriber(locale: resolvedLocale, preset: .progressiveShortDictation) do { + try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale) + } catch { + Self.debug("asset prepare failed: \(error.localizedDescription)") + continuation.yield(.error("语音语言资源未就绪，请稍后重试")) + continuation.finish() + return + } + + let newAnalyzer = SpeechAnalyzer(modules: [transcriber]) + self.lock.withLock { self.analyzer = newAnalyzer } + + guard let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat( + compatibleWith: [transcriber], + considering: Self.captureFormat + ) else { + continuation.yield(.error("当前设备不支持该语音输入格式")) + continuation.finish() + return + } + + try await newAnalyzer.prepareToAnalyze(in: analyzerFormat) + + let inputStream = self.makeInputStream(from: stream, analyzerFormat: analyzerFormat) + + // Apple recommends consuming `transcriber.results` concurrently + // while `analyzeSequence` drains the input stream. + let resultsTask = Task { + var lastText = "" for try await result in transcriber.results { if Task.isCancelled { break } - // `result.text` is an AttributedString; extract plain text. - let text = result.text.characters.map(String.init).joined() + let text = String(result.text.characters) guard !text.isEmpty, text != lastText else { continue } lastText = text continuation.yield(.partial(text)) } - } catch { - // Results sequence threw — likely cancellation. + return lastText } - if !Task.isCancelled { - continuation.yield(.final(lastText)) + let lastSampleTime = try await newAnalyzer.analyzeSequence(inputStream) + + if let lastSampleTime { + try await newAnalyzer.finalizeAndFinish(through: lastSampleTime) + } else { + try await newAnalyzer.cancelAndFinishNow() + } + + let lastText: String + do { + lastText = try await resultsTask.value + } catch { + Self.debug("transcriber results failed: \(error.localizedDescription)") + continuation.yield(.error(error.localizedDescription)) + continuation.finish() + return + } + + let trimmed = lastText.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { + continuation.yield(.error("未识别到语音内容，请重试")) + } else { + continuation.yield(.final(trimmed)) } continuation.finish() } catch is CancellationError { continuation.finish() } catch { + Self.debug("SpeechAnalyzer failed: \(error.localizedDescription)") continuation.yield(.error(error.localizedDescription)) continuation.finish() } @@ -192,64 +231,52 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable { } } - func cancel() { - let (task, currentAnalyzer) = lock.withLock { () -> (Task?, SpeechAnalyzer?) in - let t = analyzerTask - let a = analyzer - analyzerTask = nil - analyzer = nil - return (t, a) + private static func debug(_ message: String) { + #if DEBUG + print("🎙️[ASRService] \(message)") + #endif + } + + private static func prepareAssetsIfNeeded( + for transcriber: DictationTranscriber, + locale: Locale + ) async throws { + do { + _ = try await AssetInventory.reserve(locale: locale) + } catch { + // Reservation may already exist or slots are full; continue. } - task?.cancel() - if let a = currentAnalyzer { - Task { await a.cancelAndFinishNow() } + if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { + try await request.downloadAndInstall() } } - /// Maps the `AudioBufferSnapshot` stream into the `AnalyzerInput` stream - /// that `SpeechAnalyzer` consumes. - /// - /// `AudioBufferSnapshot.samples` is `[Float]` (the transport format - /// both `AudioCaptureService` and `PreviewASRController` produce — - /// Float32 is what `AVAudioEngine` gives us at the hardware rate - /// and we already downsample to 16 kHz mono before this point). - /// iOS 26's `DictationTranscriber` requires **Int16** PCM at the - /// `AnalyzerInput` boundary, so we convert per-snapshot here. - /// - /// The conversion is the textbook `[-1.0, 1.0]` × 32767 + clip + - /// cast. For a 16 kHz mono feed the loop is ~16k iters/sec — - /// well under any audio-thread budget — so a simple scalar loop - /// beats pulling in `vDSP` (which would also need a scratch - /// buffer the audio thread can't easily allocate). + func cancel() { + let (task, currentAnalyzer, finished) = lock.withLock { () -> (Task?, SpeechAnalyzer?, Bool) in + let t = analyzerTask + let a = analyzer + let f = analyzerFinished + analyzerTask = nil + analyzer = nil + return (t, a, f) + } + task?.cancel() + guard !finished, let currentAnalyzer else { return } + Task { await currentAnalyzer.cancelAndFinishNow() } + } + + /// Maps 16 kHz Float32 snapshots into `AnalyzerInput` using the format + /// returned by `bestAvailableAudioFormat(compatibleWith:considering:)`. private func makeInputStream( from stream: AsyncStream, - format: AVAudioFormat + analyzerFormat: AVAudioFormat ) -> AsyncStream { AsyncStream { continuation in Task { for await snap in stream { guard !snap.samples.isEmpty else { continue } - let capacity = AVAudioFrameCount(snap.samples.count) - guard let pcm = AVAudioPCMBuffer( - pcmFormat: format, - frameCapacity: capacity - ) else { continue } - pcm.frameLength = capacity - - // For a 1-channel Int16 buffer (interleaved or not — - // single channel, so the data layout is identical), - // `int16ChannelData?[0]` gives us the raw sample - // pointer. Clip on overflow to avoid wraparound - // (a Float like 1.5 would otherwise become a - // negative Int16 after the implicit truncation). - if let dst = pcm.int16ChannelData?[0] { - snap.samples.withUnsafeBufferPointer { src in - ASRServiceFactory.convertFloat32ToInt16( - source: src.baseAddress, - sourceCount: src.count, - destination: dst - ) - } + guard let pcm = Self.makeAnalyzerPCMBuffer(from: snap, format: analyzerFormat) else { + continue } continuation.yield(AnalyzerInput(buffer: pcm)) } @@ -257,4 +284,37 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable { } } } + + private static func makeAnalyzerPCMBuffer( + from snap: AudioBufferSnapshot, + format: AVAudioFormat + ) -> AVAudioPCMBuffer? { + let capacity = AVAudioFrameCount(snap.samples.count) + guard capacity > 0, + let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else { + return nil + } + pcm.frameLength = capacity + + switch format.commonFormat { + case .pcmFormatInt16: + guard let dst = pcm.int16ChannelData?[0] else { return nil } + snap.samples.withUnsafeBufferPointer { src in + ASRServiceFactory.convertFloat32ToInt16( + source: src.baseAddress, + sourceCount: src.count, + destination: dst + ) + } + case .pcmFormatFloat32: + guard let dst = pcm.floatChannelData?[0] else { return nil } + snap.samples.withUnsafeBufferPointer { src in + guard let base = src.baseAddress else { return } + memcpy(dst, base, src.count * MemoryLayout.stride) + } + default: + return nil + } + return pcm + } } diff --git a/OSGKeyboardShared/Services/AppGroupStore.swift b/OSGKeyboardShared/Services/AppGroupStore.swift index 505391b..e475de2 100644 --- a/OSGKeyboardShared/Services/AppGroupStore.swift +++ b/OSGKeyboardShared/Services/AppGroupStore.swift @@ -14,8 +14,15 @@ import Foundation public struct AppGroupStore: @unchecked Sendable { public let defaults: UserDefaults - public init(defaults: UserDefaults = AppGroup.defaults) { - self.defaults = defaults + public init(defaults: UserDefaults? = nil) { + if let defaults { + self.defaults = defaults + return + } + // Never hard-crash on implicit construction sites (e.g. default + // service initializers). If App Group is unavailable, use .standard + // so callers can still surface a user-facing setup error. + self.defaults = AppGroup.isAvailable ? AppGroup.defaults : .standard } // MARK: - Keys diff --git a/OSGKeyboardShared/Services/DictationBridge.swift b/OSGKeyboardShared/Services/DictationBridge.swift new file mode 100644 index 0000000..a0d3b8f --- /dev/null +++ b/OSGKeyboardShared/Services/DictationBridge.swift @@ -0,0 +1,106 @@ +// DictationBridge.swift +// OSGKeyboard · Shared +// +// Lightweight App Group bridge for host-app dictation handoff: +// keyboard extension -> open host app for recording +// host app -> writes final transcript +// keyboard extension -> consumes pending transcript and inserts text + +import Foundation + +public enum DictationBridge { + public enum Status: String, Sendable, Equatable { + case idle + case requested + case recording + case transcribing + case done + case cancelled + case error + } + + private enum Key { + static let pendingText = "dictation.pendingText" + static let updatedAt = "dictation.updatedAt" + static let status = "dictation.status" + static let statusUpdatedAt = "dictation.statusUpdatedAt" + static let statusMessage = "dictation.statusMessage" + } + + private static func resolvedDefaults(_ defaults: UserDefaults?) -> UserDefaults { + if let defaults { + return defaults + } + return AppGroup.isAvailable ? AppGroup.defaults : .standard + } + + public static func setStatus( + _ status: Status, + message: String? = nil, + defaults: UserDefaults? = nil + ) { + let store = resolvedDefaults(defaults) + store.set(status.rawValue, forKey: Key.status) + store.set(Date().timeIntervalSince1970, forKey: Key.statusUpdatedAt) + if let message, !message.isEmpty { + store.set(message, forKey: Key.statusMessage) + } else { + store.removeObject(forKey: Key.statusMessage) + } + } + + public static func currentStatus( + defaults: UserDefaults? = nil + ) -> (status: Status, message: String?, updatedAt: TimeInterval) { + let store = resolvedDefaults(defaults) + let raw = store.string(forKey: Key.status) ?? Status.idle.rawValue + let status = Status(rawValue: raw) ?? .idle + let message = store.string(forKey: Key.statusMessage) + let updatedAt = store.double(forKey: Key.statusUpdatedAt) + return (status, message, updatedAt) + } + + public static func markRequested(defaults: UserDefaults? = nil) { + let store = resolvedDefaults(defaults) + store.removeObject(forKey: Key.pendingText) + setStatus(.requested, defaults: store) + } + + /// Store a transcript for the keyboard extension to consume. + public static func storePendingTranscript(_ text: String, defaults: UserDefaults? = nil) { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + let store = resolvedDefaults(defaults) + store.set(trimmed, forKey: Key.pendingText) + store.set(Date().timeIntervalSince1970, forKey: Key.updatedAt) + setStatus(.done, defaults: store) + } + + /// Returns and clears the pending transcript if present. + public static func consumePendingTranscript( + maxAge: TimeInterval = 180, + defaults: UserDefaults? = nil + ) -> String? { + let store = resolvedDefaults(defaults) + guard let text = store.string(forKey: Key.pendingText) else { + return nil + } + if maxAge > 0 { + let ts = store.double(forKey: Key.updatedAt) + if ts > 0, Date().timeIntervalSince1970 - ts > maxAge { + clear(defaults: store) + return nil + } + } + store.removeObject(forKey: Key.pendingText) + setStatus(.idle, defaults: store) + return text + } + + public static func clear(defaults: UserDefaults? = nil) { + let store = resolvedDefaults(defaults) + store.removeObject(forKey: Key.pendingText) + store.removeObject(forKey: Key.updatedAt) + setStatus(.idle, defaults: store) + } +} diff --git a/OSGKeyboardShared/Services/FlowContinuousCapture.swift b/OSGKeyboardShared/Services/FlowContinuousCapture.swift new file mode 100644 index 0000000..40aa217 --- /dev/null +++ b/OSGKeyboardShared/Services/FlowContinuousCapture.swift @@ -0,0 +1,270 @@ +// FlowContinuousCapture.swift +// OSGKeyboard · Shared +// +// TypeWhisper-style continuous mic capture for Flow sessions: one +// AVAudioEngine + input tap for the entire session. Utterances gate +// whether buffers are forwarded to ASR; levels are always computed on +// the audio thread and read from the main thread (never UserDefaults +// from the realtime tap — that caused cross-process crashes). + +import Foundation +import AVFoundation +import os + +private enum FlowCaptureConstants { + static let levelBarCount = 24 + static let targetSampleRate: Double = 16_000 +} + +/// Thread-safe relay for utterance-scoped ASR snapshots. +private final class FlowCaptureStreamRelay: @unchecked Sendable { + private let lock = OSAllocatedUnfairLock() + private var continuation: AsyncStream.Continuation? + + func bind(_ continuation: AsyncStream.Continuation) { + lock.withLock { self.continuation = continuation } + } + + func yield(_ snapshot: AudioBufferSnapshot) { + lock.withLock { continuation?.yield(snapshot) } + } + + func finish() { + lock.withLock { + continuation?.finish() + continuation = nil + } + } +} + +/// Rolling bar levels updated from the audio tap; read on the main actor. +private final class FlowLevelStore: @unchecked Sendable { + private let lock = OSAllocatedUnfairLock() + private var levels: [Float] + + init(barCount: Int) { + levels = Array(repeating: 0, count: barCount) + } + + func update(from buffer: AVAudioPCMBuffer, barCount: Int) { + let computed = Self.calculateLevels(from: buffer, barCount: barCount) + lock.withLock { levels = computed } + } + + func snapshot() -> [Float] { + lock.withLock { levels } + } + + private static func calculateLevels(from buffer: AVAudioPCMBuffer, barCount: Int) -> [Float] { + guard let channelData = buffer.floatChannelData else { + return Array(repeating: 0, count: barCount) + } + let frameLength = Int(buffer.frameLength) + guard frameLength > 0 else { + return Array(repeating: 0, count: barCount) + } + let samplesPerBar = max(frameLength / barCount, 1) + var result = [Float]() + result.reserveCapacity(barCount) + for barIndex in 0.. 0, hwFormat.channelCount > 0 else { + throw StartError.invalidHardwareFormat( + sampleRate: hwFormat.sampleRate, + channels: Int(hwFormat.channelCount) + ) + } + + guard let targetFormat = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: FlowCaptureConstants.targetSampleRate, + channels: 1, + interleaved: false + ) else { + throw StartError.formatCreateFailed + } + guard let converter = AVAudioConverter(from: hwFormat, to: targetFormat) else { + throw StartError.converterCreateFailed + } + + if !didInstallTap { + let utteranceFlag = isUtteranceActive + let relay = streamRelay + let levels = levelStore + let tap = Self.makeAudioTapBlock( + converter: converter, + targetFormat: targetFormat, + hwFormat: hwFormat, + utteranceFlag: utteranceFlag, + levelStore: levels, + streamRelay: relay + ) + inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap) + didInstallTap = true + } + + audioEngine.prepare() + do { + try audioEngine.start() + } catch { + throw StartError.engineStartFailed(error.localizedDescription) + } + isRunning = true + } + + /// Tear down the engine and release the audio session. + public func stop() { + isUtteranceActive.withLock { $0 = false } + streamRelay.finish() + + if didInstallTap { + audioEngine.inputNode.removeTap(onBus: 0) + didInstallTap = false + } + if audioEngine.isRunning { + audioEngine.stop() + } + isRunning = false + try? AVAudioSession.sharedInstance().setActive( + false, + options: .notifyOthersOnDeactivation + ) + } + + /// Begin forwarding downsampled buffers to ASR for one utterance. + public func beginUtterance() -> AsyncStream { + isUtteranceActive.withLock { $0 = true } + let (stream, continuation) = AsyncStream.makeStream() + streamRelay.bind(continuation) + return stream + } + + /// Stop forwarding buffers; finishes the ASR stream. + public func endUtterance() { + isUtteranceActive.withLock { $0 = false } + streamRelay.finish() + } + + public func cancelUtterance() { + endUtterance() + } + + public func currentAudioLevels() -> [Float] { + levelStore.snapshot() + } + + // MARK: - Audio tap (nonisolated — runs on realtime thread) + + private nonisolated static func makeAudioTapBlock( + converter: AVAudioConverter, + targetFormat: AVAudioFormat, + hwFormat: AVAudioFormat, + utteranceFlag: OSAllocatedUnfairLock, + levelStore: FlowLevelStore, + streamRelay: FlowCaptureStreamRelay + ) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void { + return { buffer, _ in + levelStore.update(from: buffer, barCount: FlowCaptureConstants.levelBarCount) + + guard utteranceFlag.withLock({ $0 }) else { return } + + let outFrames = AVAudioFrameCount( + Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate + ) + guard outFrames > 0, + let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outFrames) + else { return } + + var error: NSError? + let status = converter.convert(to: outBuffer, error: &error) { _, outStatus in + outStatus.pointee = .haveData + return buffer + } + guard status == .haveData, error == nil, outBuffer.frameLength > 0 else { return } + + let snapshot = AudioBufferSnapshot(buffer: outBuffer) + guard !snapshot.samples.isEmpty else { return } + streamRelay.yield(snapshot) + } + } +} diff --git a/OSGKeyboardShared/Services/FlowSessionBridge.swift b/OSGKeyboardShared/Services/FlowSessionBridge.swift new file mode 100644 index 0000000..c455d64 --- /dev/null +++ b/OSGKeyboardShared/Services/FlowSessionBridge.swift @@ -0,0 +1,217 @@ +// FlowSessionBridge.swift +// OSGKeyboard · Shared +// +// TypeWhisper-style Flow session bridge: keyboard writes recording +// signals; host app writes transcription results. Legacy one-shot +// dictation handoff remains in `DictationBridge`. + +import Foundation + +public enum FlowSessionBridge { + private static func resolvedDefaults(_ defaults: UserDefaults?) -> UserDefaults { + if let defaults { return defaults } + return AppGroup.isAvailable ? AppGroup.defaults : .standard + } + + /// Force cross-process visibility. Must only be called on the main thread. + private static func flush(_ store: UserDefaults) { + if Thread.isMainThread { + store.synchronize() + } + } + + // MARK: - Session lifecycle (host app) + + public static func markSessionActive( + duration: TimeInterval = FlowSessionKeys.defaultSessionDuration, + defaults: UserDefaults? = nil + ) { + let store = resolvedDefaults(defaults) + let expires = Date().timeIntervalSince1970 + duration + store.set(true, forKey: FlowSessionKeys.flowSessionActive) + store.set(expires, forKey: FlowSessionKeys.flowSessionExpires) + writeHeartbeat(defaults: store) + setRecordingState(.idle, defaults: store) + clearTranscription(defaults: store) + flush(store) + } + + public static func markSessionInactive(defaults: UserDefaults? = nil) { + let store = resolvedDefaults(defaults) + store.set(false, forKey: FlowSessionKeys.flowSessionActive) + store.removeObject(forKey: FlowSessionKeys.flowSessionExpires) + store.removeObject(forKey: FlowSessionKeys.flowHeartbeat) + setRecordingState(.idle, defaults: store) + clearTranscription(defaults: store) + flush(store) + } + + public static func writeHeartbeat(defaults: UserDefaults? = nil) { + let store = resolvedDefaults(defaults) + store.set(Date().timeIntervalSince1970, forKey: FlowSessionKeys.flowHeartbeat) + flush(store) + } + + public static func extendSession( + by duration: TimeInterval = FlowSessionKeys.defaultSessionDuration, + defaults: UserDefaults? = nil + ) { + let store = resolvedDefaults(defaults) + let expires = Date().timeIntervalSince1970 + duration + store.set(true, forKey: FlowSessionKeys.flowSessionActive) + store.set(expires, forKey: FlowSessionKeys.flowSessionExpires) + flush(store) + } + + // MARK: - Session validity (keyboard) + + /// True when expires is in the future and heartbeat is fresh. + public static func isSessionActive(defaults: UserDefaults? = nil) -> Bool { + let store = resolvedDefaults(defaults) + flush(store) + let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires) + guard expires > Date().timeIntervalSince1970 else { return false } + + let heartbeat = store.double(forKey: FlowSessionKeys.flowHeartbeat) + guard heartbeat > 0 else { return false } + + let staleness = Date().timeIntervalSince1970 - heartbeat + return staleness <= FlowSessionKeys.heartbeatStaleInterval + } + + public static func sessionExpiresAt(defaults: UserDefaults? = nil) -> TimeInterval? { + let store = resolvedDefaults(defaults) + let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires) + return expires > 0 ? expires : nil + } + + /// Seconds until session expiry; nil when expired or never started. + public static func remainingSessionDuration(defaults: UserDefaults? = nil) -> TimeInterval? { + guard let expires = sessionExpiresAt(defaults: defaults) else { return nil } + let remaining = expires - Date().timeIntervalSince1970 + return remaining > 0 ? remaining : nil + } + + // MARK: - Recording signals (keyboard → host) + + public static func setRecordingState( + _ state: FlowSessionKeys.RecordingState, + defaults: UserDefaults? = nil + ) { + let store = resolvedDefaults(defaults) + store.set(state.rawValue, forKey: FlowSessionKeys.keyboardRecordingState) + flush(store) + } + + public static func recordingState( + defaults: UserDefaults? = nil + ) -> FlowSessionKeys.RecordingState { + let store = resolvedDefaults(defaults) + let raw = store.string(forKey: FlowSessionKeys.keyboardRecordingState) ?? FlowSessionKeys.RecordingState.idle.rawValue + return FlowSessionKeys.RecordingState(rawValue: raw) ?? .idle + } + + public static func setTranscriptionLanguage( + _ localeId: String, + defaults: UserDefaults? = nil + ) { + let store = resolvedDefaults(defaults) + store.set(localeId, forKey: FlowSessionKeys.transcriptionLanguage) + flush(store) + } + + // MARK: - Results (host → keyboard) + + public static func storeTranscriptionResult( + _ text: String, + defaults: UserDefaults? = nil + ) { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + let store = resolvedDefaults(defaults) + store.set(trimmed, forKey: FlowSessionKeys.transcriptionResult) + store.removeObject(forKey: FlowSessionKeys.transcriptionError) + setRecordingState(.idle, defaults: store) + flush(store) + } + + public static func storeTranscriptionError( + _ message: String, + defaults: UserDefaults? = nil + ) { + let store = resolvedDefaults(defaults) + store.set(message, forKey: FlowSessionKeys.transcriptionError) + setRecordingState(.idle, defaults: store) + flush(store) + } + + /// Returns and clears a pending transcription result, if any. + public static func consumeTranscriptionResult(defaults: UserDefaults? = nil) -> String? { + let store = resolvedDefaults(defaults) + flush(store) + guard let text = store.string(forKey: FlowSessionKeys.transcriptionResult), !text.isEmpty else { + return nil + } + store.removeObject(forKey: FlowSessionKeys.transcriptionResult) + flush(store) + return text + } + + /// Returns and clears a pending transcription error, if any. + public static func consumeTranscriptionError(defaults: UserDefaults? = nil) -> String? { + let store = resolvedDefaults(defaults) + flush(store) + guard let message = store.string(forKey: FlowSessionKeys.transcriptionError), !message.isEmpty else { + return nil + } + store.removeObject(forKey: FlowSessionKeys.transcriptionError) + flush(store) + return message + } + + public static func audioLevels(defaults: UserDefaults? = nil) -> [Float] { + let store = resolvedDefaults(defaults) + flush(store) + if let levels = store.array(forKey: FlowSessionKeys.audioLevels) as? [Double], !levels.isEmpty { + return levels.map { Float($0) } + } + if let levels = store.array(forKey: FlowSessionKeys.audioLevels) as? [NSNumber], !levels.isEmpty { + return levels.map { $0.floatValue } + } + return [] + } + + /// Host app: publish waveform bars for the keyboard (main thread only). + public static func storeAudioLevels( + _ levels: [Float], + defaults: UserDefaults? = nil + ) { + let store = resolvedDefaults(defaults) + store.set(levels.map { Double($0) }, forKey: FlowSessionKeys.audioLevels) + flush(store) + } + + /// Clear pending result/error before a new utterance. + public static func clearPendingTranscription(defaults: UserDefaults? = nil) { + let store = resolvedDefaults(defaults) + clearTranscription(defaults: store) + flush(store) + } + + public static func clearFlowState(defaults: UserDefaults? = nil) { + let store = resolvedDefaults(defaults) + store.set(false, forKey: FlowSessionKeys.flowSessionActive) + store.removeObject(forKey: FlowSessionKeys.flowSessionExpires) + store.removeObject(forKey: FlowSessionKeys.flowHeartbeat) + store.removeObject(forKey: FlowSessionKeys.keyboardRecordingState) + store.removeObject(forKey: FlowSessionKeys.transcriptionLanguage) + clearTranscription(defaults: store) + store.removeObject(forKey: FlowSessionKeys.audioLevels) + flush(store) + } + + private static func clearTranscription(defaults: UserDefaults) { + defaults.removeObject(forKey: FlowSessionKeys.transcriptionResult) + defaults.removeObject(forKey: FlowSessionKeys.transcriptionError) + } +} diff --git a/OSGKeyboardShared/Services/FlowSessionKeys.swift b/OSGKeyboardShared/Services/FlowSessionKeys.swift new file mode 100644 index 0000000..8f947cb --- /dev/null +++ b/OSGKeyboardShared/Services/FlowSessionKeys.swift @@ -0,0 +1,35 @@ +// FlowSessionKeys.swift +// OSGKeyboard · Shared +// +// App Group keys for TypeWhisper-style Flow sessions between the +// keyboard extension and the host app (Session Owner). + +import Foundation + +public enum FlowSessionKeys { + public static let flowSessionActive = "flow.flowSessionActive" + public static let flowSessionExpires = "flow.flowSessionExpires" + public static let flowHeartbeat = "flow.flowHeartbeat" + public static let keyboardRecordingState = "flow.keyboardRecordingState" + public static let transcriptionLanguage = "flow.transcriptionLanguage" + public static let transcriptionResult = "flow.transcriptionResult" + public static let transcriptionError = "flow.transcriptionError" + public static let audioLevels = "flow.audioLevels" + + /// Heartbeat older than this implies the host app was killed. + public static let heartbeatStaleInterval: TimeInterval = 3 + + /// Default Flow session length when started from the keyboard. + public static let defaultSessionDuration: TimeInterval = 480 + + /// Maximum duration for a single keyboard utterance. + public static let maxUtteranceDuration: TimeInterval = 60 + + public enum RecordingState: String, Sendable, Equatable { + case idle + case recording + case stopped + case processing + case aborted + } +} diff --git a/OSGKeyboardShared/Services/KeyboardState.swift b/OSGKeyboardShared/Services/KeyboardState.swift index 6f970f9..b13fdf9 100644 --- a/OSGKeyboardShared/Services/KeyboardState.swift +++ b/OSGKeyboardShared/Services/KeyboardState.swift @@ -67,6 +67,8 @@ public final class KeyboardState: ObservableObject { /// `true` — kept on the state object because the UI's status /// badge still wants a single source of truth to read from. @Published public var onDeviceSupported: Bool = false + /// Seconds remaining in the current utterance (Flow tap-to-talk). + @Published public var utteranceRemainingSeconds: Int = Int(FlowSessionKeys.maxUtteranceDuration) /// "local" → ASR only, no LLM. "cloud" → ASR + optional LLM polish. @Published public var engineMode: String = "cloud" diff --git a/OSGKeyboardShared/Services/LiveDictationController.swift b/OSGKeyboardShared/Services/LiveDictationController.swift new file mode 100644 index 0000000..bee1d52 --- /dev/null +++ b/OSGKeyboardShared/Services/LiveDictationController.swift @@ -0,0 +1,506 @@ +// LiveDictationController.swift +// OSGKeyboard · Shared +// +// Unified on-device dictation session: mic capture + iOS 26 SpeechAnalyzer. +// Used by the keyboard preview sheet, host-app dictation handoff, and any +// other foreground surface that needs live ASR without duplicating pipeline code. +// Owns its own AVAudioEngine + AVAudioSession, downsamples to 16 kHz +// mono Float32 on the audio thread (same as `AudioCaptureService`), and +// feeds `AudioBufferSnapshot` to the shared `ASRService` (the same +// pipeline the real keyboard extension +// uses, so the preview exercises the *real* iOS speech APIs, not a +// stub). Without this the in-app preview was a hardcoded transcript +// and "did you actually call SFSpeechRecognizer?" was a fair review +// note. +// +// Why not reuse `AudioCaptureService` from the extension? It lives in +// `OSGKeyboardExt`, an `app-extension` target — the main app can't +// import its symbols. We could move it to `OSGKeyboardShared`, but +// `AVAudioSession` lifecycle differs enough between a keyboard +// extension (no background, no recording entitlement surprise) and a +// foreground app that a copy here is the lesser evil. + +import Foundation +import AVFoundation +import Speech +import os + +/// Thread-safe relay so the AVAudioEngine tap can yield snapshots without +/// hopping through `@MainActor` (which adds latency and can reorder frames). +private final class CaptureStreamRelay: @unchecked Sendable { + private let lock = OSAllocatedUnfairLock() + private var continuation: AsyncStream.Continuation? + + func bind(_ continuation: AsyncStream.Continuation) { + lock.withLock { self.continuation = continuation } + } + + func yield(_ snapshot: AudioBufferSnapshot) { + lock.withLock { continuation?.yield(snapshot) } + } + + func finish() { + lock.withLock { + continuation?.finish() + continuation = nil + } + } +} + +@MainActor +public final class LiveDictationController: ObservableObject { + + public enum Phase: Equatable { + case idle + case requestingPermission + case recording + case processing + case denied(String) + case error(String) + } + + @Published public private(set) var phase: Phase = .idle + /// Normalized 0...1 RMS for the disc level meter. Polled from the + /// audio tap via `Task { @MainActor in ... }` — the tap itself + /// runs on a real-time audio thread, so we never touch published + /// state from there. + @Published public private(set) var level: Double = 0 + @Published public private(set) var currentPartial: String = "" + @Published public private(set) var errorMessage: String? + + /// Set when a `.final` ASR event lands. The owning sheet observes + /// this and appends the text to its textbox, then clears it so the + /// next recording starts from zero. + @Published public var lastFinal: String = "" + + private let asr: ASRService = ASRServiceFactory.make() + private let audioEngine = AVAudioEngine() + /// `internal` (not `private`) so the regression test in + /// `OSGKeyboardTests/PreviewASRControllerStateTests.swift` can + /// install a known consumer task and assert `stop()` doesn't + /// cancel it. The class is `@MainActor`-isolated, so the + /// natural Swift 6 isolation rules still prevent production + /// code outside the class from racing on it. + public var asrTask: Task? + private let streamRelay = CaptureStreamRelay() + private var didConfigureAudioSession = false + private var didInstallTap = false + + public init() {} + + /// Start dictation using a persisted settings locale id (`auto`, `zh-Hans`, …). + public func start(localeId: String) async { + await start(locale: SpeechLocaleResolver.resolve(localeId)) + } + + public func start(locale: Locale) async { + // Re-entry guard: ignore taps that arrive while we're already + // running. (The sheet's `cyclePhase` is also guarded, but + // async race windows are easier to lock down here.) + switch phase { + case .recording, .requestingPermission, .processing: + return + default: + break + } + // Cancel any leftover consumer task from a previous recording. + // Normally `stop()` lets the task run to completion (so it can + // see the `.final` and transition out of `.processing`), but if + // the user smashed the disc twice — stop, then immediately + // start — the previous task might still be draining. Cancel it + // here so we don't have two consumer tasks fighting over the + // same `events` stream. + asrTask?.cancel() + asrTask = nil + teardownCapturePipeline() + phase = .requestingPermission + currentPartial = "" + lastFinal = "" + errorMessage = nil + level = 0 + + // 1. Microphone permission. The helper is `nonisolated` so the + // (iOS < 17) callback closure does not inherit `@MainActor` — + // `AVAudioSession.requestRecordPermission` delivers on a TCC + // reply queue, and a `@MainActor`-inferred closure body there + // hits `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift`. + let micGranted = await Self.requestMicrophonePermission() + guard micGranted else { + phase = .denied(NSLocalizedString("keyboard.denied.mic", comment: "")) + return + } + + // 2. Speech recognition permission. Same reasoning as above: + // the callback fires on TCC's reply queue, NOT the main queue. + let speechGranted = await Self.requestSpeechRecognitionPermission() + guard speechGranted else { + phase = .denied(NSLocalizedString("keyboard.denied.speech", comment: "")) + return + } + + // 3. Audio session — only configure once per process. + // + // Category is `.record` (not `.playAndRecord`) because the + // preview never plays back audio — it just records from the + // mic and hands the buffers to `SpeechAnalyzer`. On the + // iOS Simulator, `.playAndRecord` requires the + // `AURemoteIO` Audio Unit's *output* side to also be + // enabled, but the simulator's "speaker" reports a 0 Hz + // hardware format, so `AURemoteIO::enable` fails with + // `kAudioUnitErr_FormatNotSupported` (-10851) and any + // subsequent `installTap` traps with "Failed to create tap + // due to format mismatch". `.record` skips the output + // side entirely, so the simulator can record. + // + // The real keyboard extension (`OSGKeyboardExt`) keeps + // `.playAndRecord` because it runs on a real device where + // the output side has a real hardware format, and may want + // to play click sounds / haptic feedback. Only the preview + // needs the simulator-friendly category. + if !didConfigureAudioSession { + do { + let session = AVAudioSession.sharedInstance() + try session.setCategory(.record, + mode: .measurement, + options: []) + try session.setActive(true, options: .notifyOthersOnDeactivation) + didConfigureAudioSession = true + } catch { + debug("audio session failed: \(error.localizedDescription)") + phase = .error(String.localizedStringWithFormat( + NSLocalizedString("preview.error.audioSession", comment: ""), + error.localizedDescription + )) + return + } + } + + // 4. Spin up the engine + ASR. + phase = .recording + startEngineAndASR(locale: locale) + } + + public func stop() { + // Don't `asrTask?.cancel()` here — see the comment in + // `startEngineAndASR` for the full rationale. Short version: + // cancelling the consumer task at the same moment we close the + // audio stream also triggers the producer's + // `continuation.onTermination → self?.cancel()` cascade, which + // marks the producer's outer task as cancelled and skips the + // `.final` event. The UI is then left in `.processing` forever + // because no one schedules the transition out. The consumer + // task naturally exits when `events` finishes, so the right + // thing is to let it run. + // + // If a previous `asrTask` is somehow still running (e.g. the + // user smashed the disc twice quickly), `start()` cancels it + // at the entry point as a safety net. + teardownCapturePipeline() + // Fallback: if we already have a meaningful partial but the + // backend never emits `.final`, promote the partial so the + // preview still inserts text after "停止录音". + let partial = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines) + if !partial.isEmpty && lastFinal.isEmpty { + lastFinal = partial + currentPartial = "" + } + if phase == .recording { + phase = .processing + } + // Deactivate so the user's music resumes if the preview is + // dismissed mid-recording. + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + + // Safety net: if the ASR pipeline never produces a `.final` + // (analyzer hang, system glitch, dropped continuation), force + // the UI back to idle after a short delay so the user isn't + // stuck. Normal recordings complete well under a second, so + // the 3-second budget is only hit on the unhappy path; if the + // pipeline finishes first and flips the phase to `.idle` (or + // `.error`), the check below no-ops. + Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(3)) + guard let self else { return } + if self.phase == .processing { + let stalePartial = self.currentPartial.trimmingCharacters(in: .whitespacesAndNewlines) + if !stalePartial.isEmpty, self.lastFinal.isEmpty { + self.debug("processing timeout, using partial") + self.lastFinal = stalePartial + self.currentPartial = "" + } + self.phase = .idle + } + } + } + + public func reset() { + // Called by the sheet after appending `lastFinal` to the textbox, + // so the next recording can produce a fresh final without us + // double-appending. + lastFinal = "" + if phase == .processing { + phase = .idle + } + } + + // MARK: - Engine + ASR + + private func startEngineAndASR(locale: Locale) { + let inputNode = audioEngine.inputNode + let hwFormat = inputNode.outputFormat(forBus: 0) + + // Pre-flight check: a placeholder / unconfigured input bus + // reports `sampleRate == 0` (or `channelCount == 0`). + // `installTap` on such a bus traps with "Failed to create + // tap due to format mismatch" (an NSException, not a Swift + // `Error`, so we can't `try`/`catch` it). The safest fix + // is to refuse the tap up front and surface a clear + // `.error` phase instead of crashing the app. We've seen + // this on the iOS Simulator when the host's microphone + // permission isn't granted to CoreSimulator, and on + // devices where the audio session is in an unexpected + // state from a previous foreground/background transition. + guard hwFormat.sampleRate > 0, hwFormat.channelCount > 0 else { + debug("invalid hardware format sr=\(hwFormat.sampleRate) ch=\(hwFormat.channelCount)") + phase = .error( + String.localizedStringWithFormat( + NSLocalizedString("preview.error.micUnavailable", comment: ""), + hwFormat.sampleRate, + Int(hwFormat.channelCount) + ) + ) + return + } + + let targetSampleRate: Double = 16_000 + guard let targetFormat = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: targetSampleRate, + channels: 1, + interleaved: false + ) else { + phase = .error(NSLocalizedString("preview.error.formatCreate", comment: "")) + return + } + guard let converter = AVAudioConverter(from: hwFormat, to: targetFormat) else { + debug("converter creation failed") + phase = .error(NSLocalizedString("preview.error.converterCreate", comment: "")) + return + } + + let (stream, continuation) = AsyncStream.makeStream() + streamRelay.bind(continuation) + + // Tap the hardware input. The closure passed to `installTap` runs + // on the AVAudioEngine real-time audio thread. In Swift 6 strict + // concurrency, a closure literal defined inside a `@MainActor` + // method inherits `@MainActor` isolation, which would trip + // `dispatch_assert_queue_fail` on first invocation from the + // audio thread. The fix is to build the actual tap body in a + // `nonisolated` helper (`makeAudioTapBlock`) and have the + // installTap closure be a single function reference — function + // references never carry inferred isolation, so the dispatch + // runtime is happy and the body runs wherever AVAudioEngine + // wants it (the audio thread). + let onMeter: @Sendable (Double) -> Void = { [weak self] meter in + Task { @MainActor [weak self] in + guard let self else { return } + // Lightweight smoothing so the disc ring doesn't jitter. + self.level = self.level * 0.55 + meter * 0.45 + } + } + let relay = streamRelay + let onSnapshot: @Sendable (AudioBufferSnapshot) -> Void = { snapshot in + relay.yield(snapshot) + } + let tap = Self.makeAudioTapBlock( + converter: converter, + targetFormat: targetFormat, + hwFormat: hwFormat, + onMeter: onMeter, + onSnapshot: onSnapshot + ) + inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap) + didInstallTap = true + + audioEngine.prepare() + do { + try audioEngine.start() + } catch { + debug("audio engine start failed: \(error.localizedDescription)") + phase = .error(String.localizedStringWithFormat( + NSLocalizedString("preview.error.engineStart", comment: ""), + error.localizedDescription + )) + return + } + + // 5. Wire up ASR. + let events = asr.transcribe( + stream: stream, + locale: locale + ) + asrTask = Task { @MainActor [weak self] in + guard let self else { return } + for await event in events { + switch event { + case .capability: + break + case .partial(let s): + self.currentPartial = s + case .final(let s): + let trimmed = s.trimmingCharacters(in: .whitespacesAndNewlines) + self.lastFinal = trimmed + self.currentPartial = "" + self.phase = .idle + case .error(let m): + self.debug("asr error: \(m)") + self.teardownCapturePipeline() + self.errorMessage = m + self.phase = .error(m) + } + } + } + } + + // MARK: - Permission helpers (nonisolated) + // + // `SFSpeechRecognizer.requestAuthorization` delivers its callback + // on a TCC reply queue, NOT the main queue. If we wrap that + // callback inline in `start(locale:)` — which is `@MainActor` — + // Swift 6 strict concurrency infers the closure body as + // `@MainActor`, and the runtime crashes on + // `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift` as + // soon as TCC calls us back. + // + // The first attempt (commit `e8a0310`) extracted the entire + // permission request into a `nonisolated static func` helper. + // That worked in isolation, but the Swift 6 optimizer + // inlined those helpers back into `start(locale:)`. After + // inlining, the `withCheckedContinuation` body and the + // `requestAuthorization` callback were re-typed in the + // `@MainActor` context of the caller, and the runtime + // assertion came right back — same crash, different symbol: + // `closure #1 in closure #2 in PreviewASRController.start(locale:)`. + // + // The fix that survives inlining is the *function-reference* + // pattern, the same one used for `installTap` in + // `makeAudioTapBlock` below. The callback is built in a + // `nonisolated` static helper that takes a `CheckedContinuation` + // and returns the `(Status) -> Void` handler. The body of that + // helper has no enclosing actor, so the closure is created in + // nonisolated context. When TCC calls us back, the runtime + // sees a nonisolated closure on a non-main queue and is happy. + // + // `cont.resume(...)` is itself thread-safe on + // `CheckedContinuation`, so we don't need to hop back to the + // main actor before resuming. + + private nonisolated static func requestMicrophonePermission() async -> Bool { + // iOS 17+ API; the iOS < 17 fallback (`AVAudioSession.recordPermission` + // + `requestRecordPermission` callback) is gone now that the + // deployment target is iOS 26. + switch AVAudioApplication.shared.recordPermission { + case .granted: return true + case .denied: return false + case .undetermined: return await AVAudioApplication.requestRecordPermission() + @unknown default: return false + } + } + + private nonisolated static func requestSpeechRecognitionPermission() async -> Bool { + await withCheckedContinuation { (cont: CheckedContinuation) in + SFSpeechRecognizer.requestAuthorization( + Self.makeSpeechAuthHandler(continuation: cont) + ) + } + } + + private nonisolated static func makeSpeechAuthHandler( + continuation: CheckedContinuation + ) -> @Sendable (SFSpeechRecognizerAuthorizationStatus) -> Void { + return { status in + continuation.resume(returning: status == .authorized) + } + } + + // MARK: - Audio tap (nonisolated, runs on AVAudioEngine render thread) + // + // `AVAudioNode.installTap`'s callback fires on the audio engine's + // real-time render thread. In Swift 6 strict concurrency, a closure + // literal defined inside a `@MainActor` method inherits `@MainActor` + // isolation — and `dispatch_assert_queue_fail` fires the moment + // the runtime tries to dispatch that closure on a non-main queue. + // + // The trick is to build the actual tap body in a `nonisolated` + // function and have the installTap closure be a *function reference* + // to that helper. Function references never carry inferred + // isolation, so the dispatch runtime is satisfied and the body + // runs wherever AVAudioEngine wants. State updates to + // `self.level` and the AsyncStream continuation hop back to the + // main actor via `Task { @MainActor in … }`, which is itself + // safe to call from a non-isolated context. + private nonisolated static func makeAudioTapBlock( + converter: AVAudioConverter, + targetFormat: AVAudioFormat, + hwFormat: AVAudioFormat, + onMeter: @Sendable @escaping (Double) -> Void, + onSnapshot: @Sendable @escaping (AudioBufferSnapshot) -> Void + ) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void { + // `@Sendable` on the returned closure makes the Sendable + // conformance explicit. `AVAudioNodeTapBlock` is declared as + // a plain escaping closure in the SDK; we cast at the call + // site via `as @Sendable`. + return { buffer, _ in + // 1) Level meter from raw hardware buffer. + let n = Int(buffer.frameLength) + var sumSquares: Float = 0 + if let channelData = buffer.floatChannelData?[0], n > 0 { + for i in 0.. 0 ? sqrtf(sumSquares / Float(n)) : 0 + let meter = min(Double(rms) * 4.0, 1.0) + onMeter(meter) + + // 2) Downsample to 16 kHz mono Float32 for ASR (matches + // `AudioCaptureService` and Apple's `considering:` hint). + let outFrames = AVAudioFrameCount( + Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate + ) + guard outFrames > 0, + let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outFrames) + else { return } + + var error: NSError? + let status = converter.convert(to: outBuffer, error: &error) { _, outStatus in + outStatus.pointee = .haveData + return buffer + } + guard status == .haveData, error == nil, outBuffer.frameLength > 0 else { return } + + let snapshot = AudioBufferSnapshot(buffer: outBuffer) + guard !snapshot.samples.isEmpty else { return } + onSnapshot(snapshot) + } + } + + private func teardownCapturePipeline() { + if didInstallTap { + audioEngine.inputNode.removeTap(onBus: 0) + didInstallTap = false + } + if audioEngine.isRunning { + audioEngine.stop() + } + streamRelay.finish() + } + + private func debug(_ message: String) { + #if DEBUG + print("🎙️[LiveDictationController] \(message)") + #endif + } +} diff --git a/OSGKeyboardShared/Services/SpeechLocaleResolver.swift b/OSGKeyboardShared/Services/SpeechLocaleResolver.swift new file mode 100644 index 0000000..e649b35 --- /dev/null +++ b/OSGKeyboardShared/Services/SpeechLocaleResolver.swift @@ -0,0 +1,25 @@ +// SpeechLocaleResolver.swift +// OSGKeyboard · Shared +// +// Maps persisted `localeId` settings to a `Locale` suitable for +// `DictationTranscriber.supportedLocale(equivalentTo:)`. + +import Foundation + +public enum SpeechLocaleResolver { + /// Resolve a stored locale id (`auto`, `zh-Hans`, …) for on-device ASR. + public static func resolve(_ localeId: String) -> Locale { + let raw: String + if localeId == "auto" { + raw = Locale.preferredLanguages.first ?? "en-US" + } else { + raw = localeId + } + let normalized = raw.replacingOccurrences(of: "_", with: "-").lowercased() + if normalized.hasPrefix("zh") { return Locale(identifier: "zh-Hans") } + if normalized.hasPrefix("ja") { return Locale(identifier: "ja-JP") } + if normalized.hasPrefix("ko") { return Locale(identifier: "ko-KR") } + if normalized.hasPrefix("en") { return Locale(identifier: "en-US") } + return Locale(identifier: raw.replacingOccurrences(of: "_", with: "-")) + } +} diff --git a/OSGKeyboardShared/Utilities/DictationTextComposer.swift b/OSGKeyboardShared/Utilities/DictationTextComposer.swift new file mode 100644 index 0000000..f3b1313 --- /dev/null +++ b/OSGKeyboardShared/Utilities/DictationTextComposer.swift @@ -0,0 +1,17 @@ +// DictationTextComposer.swift +// OSGKeyboard · Shared +// +// Merges pre-dictation anchor text with a live cumulative transcript. + +import Foundation + +public enum DictationTextComposer { + /// Combine text that existed before dictation with the current live transcript. + public static func compose(anchor: String, live: String) -> String { + let trimmed = live.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return anchor } + if anchor.isEmpty { return trimmed } + if anchor.last == " " || anchor.last == "\n" { return anchor + trimmed } + return anchor + " " + trimmed + } +} diff --git a/OSGKeyboardTests/DictationBridgeTests.swift b/OSGKeyboardTests/DictationBridgeTests.swift new file mode 100644 index 0000000..db85039 --- /dev/null +++ b/OSGKeyboardTests/DictationBridgeTests.swift @@ -0,0 +1,59 @@ +// DictationBridgeTests.swift +// OSGKeyboardTests + +import XCTest +@testable import OSGKeyboardShared + +final class DictationBridgeTests: XCTestCase { + + private func makeDefaults() -> UserDefaults { + let suite = "group.com.osgkeyboard.shared.tests.dictation.\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suite)! + defaults.removePersistentDomain(forName: suite) + return defaults + } + + func testStoreAndConsumeTranscript() { + let defaults = makeDefaults() + + DictationBridge.storePendingTranscript(" hello ", defaults: defaults) + let consumed = DictationBridge.consumePendingTranscript(defaults: defaults) + + XCTAssertEqual(consumed, "hello") + XCTAssertNil(DictationBridge.consumePendingTranscript(defaults: defaults)) + } + + func testConsumeIgnoresExpiredTranscript() { + let defaults = makeDefaults() + DictationBridge.storePendingTranscript("stale", defaults: defaults) + // maxAge = 1ms, then delay to force expiry + usleep(2_000) + let consumed = DictationBridge.consumePendingTranscript(maxAge: 0.001, defaults: defaults) + XCTAssertNil(consumed) + } + + func testStatusLifecycle() { + let defaults = makeDefaults() + + DictationBridge.markRequested(defaults: defaults) + XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .requested) + + DictationBridge.setStatus(.recording, defaults: defaults) + XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .recording) + + DictationBridge.storePendingTranscript("ok", defaults: defaults) + XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .done) + + _ = DictationBridge.consumePendingTranscript(defaults: defaults) + XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .idle) + } + + func testStatusMessageAndTimestamp() { + let defaults = makeDefaults() + DictationBridge.setStatus(.error, message: "fail", defaults: defaults) + let snapshot = DictationBridge.currentStatus(defaults: defaults) + XCTAssertEqual(snapshot.status, .error) + XCTAssertEqual(snapshot.message, "fail") + XCTAssertGreaterThan(snapshot.updatedAt, 0) + } +} diff --git a/OSGKeyboardTests/FlowSessionBridgeTests.swift b/OSGKeyboardTests/FlowSessionBridgeTests.swift new file mode 100644 index 0000000..6790276 --- /dev/null +++ b/OSGKeyboardTests/FlowSessionBridgeTests.swift @@ -0,0 +1,52 @@ +// FlowSessionBridgeTests.swift +// OSGKeyboardTests + +import XCTest +@testable import OSGKeyboardShared + +final class FlowSessionBridgeTests: XCTestCase { + + private func makeDefaults() -> UserDefaults { + let suite = "group.com.osgkeyboard.shared.tests.flow.\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suite)! + defaults.removePersistentDomain(forName: suite) + return defaults + } + + func testSessionActiveRequiresFreshHeartbeat() { + let defaults = makeDefaults() + FlowSessionBridge.markSessionActive(duration: 60, defaults: defaults) + XCTAssertTrue(FlowSessionBridge.isSessionActive(defaults: defaults)) + + let staleHeartbeat = Date().timeIntervalSince1970 - 10 + defaults.set(staleHeartbeat, forKey: FlowSessionKeys.flowHeartbeat) + XCTAssertFalse(FlowSessionBridge.isSessionActive(defaults: defaults)) + } + + func testRecordingStateRoundTrip() { + let defaults = makeDefaults() + FlowSessionBridge.setRecordingState(.recording, defaults: defaults) + XCTAssertEqual(FlowSessionBridge.recordingState(defaults: defaults), .recording) + + FlowSessionBridge.setRecordingState(.stopped, defaults: defaults) + XCTAssertEqual(FlowSessionBridge.recordingState(defaults: defaults), .stopped) + } + + func testConsumeTranscriptionResultClearsKey() { + let defaults = makeDefaults() + FlowSessionBridge.storeTranscriptionResult("hello", defaults: defaults) + XCTAssertEqual(FlowSessionBridge.consumeTranscriptionResult(defaults: defaults), "hello") + XCTAssertNil(FlowSessionBridge.consumeTranscriptionResult(defaults: defaults)) + } + + func testClearFlowStateRemovesSessionKeys() { + let defaults = makeDefaults() + FlowSessionBridge.markSessionActive(defaults: defaults) + FlowSessionBridge.storeTranscriptionResult("x", defaults: defaults) + FlowSessionBridge.clearFlowState(defaults: defaults) + + XCTAssertFalse(defaults.bool(forKey: FlowSessionKeys.flowSessionActive)) + XCTAssertNil(FlowSessionBridge.consumeTranscriptionResult(defaults: defaults)) + XCTAssertEqual(FlowSessionBridge.recordingState(defaults: defaults), .idle) + } +} diff --git a/OSGKeyboardTests/PreviewASRControllerStateTests.swift b/OSGKeyboardTests/PreviewASRControllerStateTests.swift index d62ade2..bf5d3ef 100644 --- a/OSGKeyboardTests/PreviewASRControllerStateTests.swift +++ b/OSGKeyboardTests/PreviewASRControllerStateTests.swift @@ -29,8 +29,7 @@ // the ASR to never yield `.final` and asserts the timeout fires. import XCTest -@testable import OSGKeyboard -import AVFoundation +@testable import OSGKeyboardShared @MainActor final class PreviewASRControllerStateTests: XCTestCase { @@ -41,7 +40,7 @@ final class PreviewASRControllerStateTests: XCTestCase { /// is the regression test for the original bug — if it ever /// fails, the disc is stuck again. func testStopDoesNotCancelConsumerTask() { - let controller = PreviewASRController() + let controller = LiveDictationController() let consumerTask = Task {} controller.asrTask = consumerTask @@ -58,7 +57,7 @@ final class PreviewASRControllerStateTests: XCTestCase { /// misbehave — no double-cancel, no extra state transitions. /// This is the contract `.onDisappear` relies on. func testStopIsIdempotent() { - let controller = PreviewASRController() + let controller = LiveDictationController() let consumerTask = Task {} controller.asrTask = consumerTask diff --git a/TYPEWHISPER_FLOW_MIGRATION_TRACKER.md b/TYPEWHISPER_FLOW_MIGRATION_TRACKER.md new file mode 100644 index 0000000..421f6d4 --- /dev/null +++ b/TYPEWHISPER_FLOW_MIGRATION_TRACKER.md @@ -0,0 +1,270 @@ +# OSGKeyboard · TypeWhisper Flow 迁移蓝图与任务追踪 + +> 目标：把当前“每次按语音都尝试跳转主 App”的模式，迁移为“Flow Session 会话模式”，实现**仅键盘侧连续语音输入**（会话有效期间无需反复跳转）。 +> +> 维护方式：每完成一个任务，把对应复选框从 `[ ]` 改为 `[x]`，并填写“完成记录”。 + +--- + +## 1) 迁移蓝图（最终目标架构） + +### 1.1 第一性目标 +- 主链路不依赖每次 `openHostApp(dictate)` 成功。 +- 会话有效期间，键盘只做“开始/停止”信号与结果插入。 +- 跳转主 App 降级为“会话初始化/修复”路径。 +- 任何异常都可恢复，不出现卡死状态。 + +### 1.2 目标架构 +- **主 App（Session Owner）** + - 维护 Flow 会话生命周期（active/expired/inactive） + - 持续写心跳（heartbeat） + - **持有唯一 continuous 音频管线**（见 §1.5） + - 处理键盘录音状态信号并执行识别 + - 回写 `transcriptionResult/transcriptionError` +- **键盘扩展（Signal + Insert）** + - 判断会话是否有效（active + expires + heartbeat） + - 会话有效：写 `recording/stopped/aborted` + - 会话无效：引导启动会话（一次性） + - 轮询结果并 `insertText` +- **App Group（单一事实源）** + - 所有跨进程状态仅通过共享键传递 + +### 1.3 关键共享键 +- `flowSessionActive: Bool` +- `flowSessionExpires: TimeInterval` +- `flowHeartbeat: TimeInterval` +- `keyboardRecordingState: String` (`idle|recording|stopped|processing|aborted`) +- `transcriptionLanguage: String` +- `transcriptionResult: String` +- `transcriptionError: String` +- `audioLevels: [Float]`（键盘波形，**Phase 1 必做**） + +### 1.4 状态机约束 +- `recording` 仅在 `flowSessionActive = true` 且会话未过期时生效。 +- `stopped` 必须最终归并到 `done|error|idle`。 +- 任意异常必须显式写 `transcriptionError` 并回到 `idle`。 + +### 1.5 音频不变量（不可违背 — 对齐 TypeWhisper / SwiftSpeak SwiftLink） + +> **历史教训**：曾用 `.playback` 静音保活 + utterance 时 `LiveDictationController.start()` 临时开麦，导致 `Session activation failed` 与双 engine 崩溃。**禁止再使用该模式。** + +| # | 规则 | TypeWhisper 对应 | OSGKeyboard 实现 | +|---|------|------------------|------------------| +| A1 | 会话启动时配置 **`.playAndRecord`**（`mode: .measurement`），`setActive(true)` **一次** | `startFlowSession()` | `FlowContinuousCapture.start()` | +| A2 | 立刻 **`startContinuousRecording()`**：一个 `AVAudioEngine` + **常驻** `inputNode` tap | `startContinuousRecording()` | 同上，tap 在 `start()` 安装 | +| A3 | utterance 期间 **禁止** stop/start engine、**禁止** deactivate/reactivate session | `isRecordingAtomic` gating | `isUtteranceActive` + `beginUtterance`/`endUtterance` | +| A4 | 键盘 `recording` → 只 flip 标志 + 启动 ASR consumer；`stopped` → 结束 ASR stream + finalize | `checkKeyboardSignal()` | `FlowSessionManager.handleKeyboardSignal()` | +| A5 | **`audioLevels` 从 tap 计算**，主线程写入 App Group；**禁止**在 audio realtime 线程 `UserDefaults.synchronize()` | tap 写 levels（我们改为 main 线程 flush 更安全） | `FlowLevelStore` + `startLevelPublishing()` | +| A6 | Flow 期间 **禁止** 调用 `LiveDictationController.start()`（预览 / legacy dictate 专用） | `AudioRecordingService` 拒绝 Flow active | `FlowSessionManager` 直接用 `ASRService` | +| A7 | 会话结束才 `removeTap` / `engine.stop()` / `setActive(false)` | `endFlowSession()` | `FlowSessionManager.endSession()` | + +**错误模式（已废弃，勿恢复）：** +- ❌ `.playback` + 静音 `AVAudioPlayerNode` 保活 +- ❌ utterance 时 stop 保活 engine 再开第二个 engine 录音 +- ❌ 复用 `LiveDictationController` 作为 Flow utterance 入口 + +**参考项目分工：** +- **TypeWhisper**（主参考）：`FlowSessionManager` + continuous tap + `SFSpeechAudioBufferRecognitionRequest` / batch +- **SwiftSpeak SwiftLink**（辅参考）：Darwin 通知、前台启动约束、streaming/batch 分叉 +- **Legacy dictate**（保留）：`DictationCaptureView` + `LiveDictationController`，单次 handoff + +### 1.6 组件职责 + +| 组件 | 职责 | +|------|------| +| `FlowContinuousCapture` | 会话级 engine + tap + utterance gating + levels | +| `FlowSessionManager` | 生命周期、轮询、ASR finalize、可选 polish | +| `FlowSessionBridge` | App Group 读写 | +| `LiveDictationController` | 主 App 预览、legacy `dictate` — **不用于 Flow** | +| `KeyboardViewController` | 信号 + 轮询 + insertText | + +--- + +## 2) 详细改动清单（按文件） + +## Phase 1 · 基础设施（会话能力） + +### `OSGKeyboardShared/Services/FlowSessionBridge.swift` +- [x] Flow 键读写封装（active/expires/heartbeat/recordingState/result/error） +- [x] `storeAudioLevels` / `clearPendingTranscription` +- [x] 保留并兼容现有 `DictationBridge` pending transcript 接口 +- [x] `clearFlowState()` + +### `OSGKeyboardShared/Services/FlowContinuousCapture.swift`（新增） +- [x] `.playAndRecord` + 常驻 input tap +- [x] utterance gating → `AsyncStream` +- [x] `FlowLevelStore`（audio 线程写、main 线程读） + +### `OSGKeyboard/Services/FlowSessionManager.swift` +- [x] 会话生命周期 + heartbeat + 过期 +- [x] 轮询 `keyboardRecordingState` → utterance gating → `ASRService` +- [x] 回写 `transcriptionResult/transcriptionError` +- [x] 主线程发布 `audioLevels` +- [x] **不再**使用 playback keep-alive / `LiveDictationController` for Flow + +### `OSGKeyboard/Info.plist` 与 `project.yml` +- [x] `UIBackgroundModes: audio` +- [x] 麦克风 / 语音识别权限说明 + +--- + +## Phase 2 · 键盘主链路切换 + +### `OSGKeyboardExt/KeyboardViewController.swift` +- [x] `pressBegan()` 会话判断分流 +- [x] 会话有效：写 `recording`；无效：`openHostApp(startflow)` 或提示 +- [x] `pressEnded()` 写 `stopped` +- [x] 结果轮询 + `insertText` +- [ ] `KeyboardRootView` 会话 UI 细化（可选） + +--- + +## Phase 3 · 体验与稳定性强化 + +### 主 App / 键盘协同 +- [x] 心跳超时判死（`FlowSessionBridge.isSessionActive`） +- [x] 识别超时保护（30s finalize） +- [x] `audioLevels` 共享 +- [ ] Darwin 通知（SwiftSpeak 模式，降低轮询延迟） +- [ ] 会话死掉后一键重启 UI +- [ ] `AudioRouteCoordinator`（蓝牙/路由切换） + +### 测试 +- [x] App Group 状态机单测 +- [ ] Flow continuous capture 单测（需 device / mock） +- [ ] 端到端真机回归 + +--- + +## 3) 任务追踪面板 + +## A. 已完成 +- [x] A1–A4：架构研究、蓝图、追踪文档 +- [x] B1：Phase 1 Flow 基础设施（含音频层修正） +- [x] B2：Phase 2 键盘主链路（核心路径） + +## B. 待执行 +- [ ] B3：Phase 3 体验增强（Darwin、路由、恢复 UI）→ 见 §7.7 批次 F +- [ ] B4：端到端真机回归 → 见 §7.7；本地/在线主路径已通过 +- [ ] **Phase 4**：§7.2–7.7 批次 A–F + +--- + +## 5) 验收标准（Definition of Done） + +### Flow 核心（Phase 1–2） +- [x] 本地 / 云端模式均可回填（真机已验证） +- [ ] 连续 20 次语音输入，会话有效期间 **无需** 反复跳主 App +- [ ] Console **无** `Session activation failed` / playback↔record 循环 +- [ ] 键盘波形随说话变化（`audioLevels` 非零） +- [ ] 微信/备忘录/Safari 稳定回填 + +### Phase 4 新增 +- [ ] 打开 App 后 **自动** 语音会话（权限齐全时） +- [ ] 杀 App 再开 → **冷启动恢复**（未过期） +- [ ] 键盘 **点按** 开始/结束；60s 倒计时 + 最后 10s 变红 +- [ ] Onboarding **分步权限** 完整可走通 +- [ ] 隐私政策 URL 可访问；App 内可打开 +- [ ] App Store 隐私标签与政策一致 +- [ ] 单测覆盖核心状态迁移 + +--- + +## 6) 真机验证清单（最小） + +1. 打开 App → **自动**进入语音会话（或 Home 卡片显示「进行中」） +2. 切备忘录 → **点按**键盘麦开始 → 再点结束 → **文字出现** +3. **不跳主 App**，重复 5 次 +4. 本地模式 + 云端模式各测 1 次 +5. 60s 倒计时 + 最后 10 秒变红；到点自动识别 +6. 杀 App 再开 → 会话恢复（未过期时） +7. 预览 sheet「开始/停止录音」仍可用（Flow 未启动时） + +--- + +## 7) Phase 4 · 产品体验与上架合规（2026-06 拍板） + +> 以下为用户/产品确认规格。**实现顺序建议：B → C → D → A → E → F。** + +### 7.1 已锁定决策 + +| 主题 | 决定 | +|------|------| +| 麦克风交互 | **点按开始 / 再点结束**；识别中不可取消 | +| 会话未启动 | 保持现有逻辑（拉 App / 提示去主 App） | +| 权限引导 | 欢迎 → 麦克风 → 语音识别 → 键盘+完全访问 → 引擎/API；**仅首次或权限未定时** | +| 自动开语音会话 | 进 App 且权限齐全 → **自动开**；有效会话 → **续期**；**不设关闭开关** | +| 冷启动恢复 | 参考 TypeWhisper `checkExistingSession` | +| Home 按钮 | 自动开后 **隐藏「启动」**；**保留「结束」**；失败显示原因 + 去设置 | +| 单次录音上限 | **60s**；键盘到点 auto `stopped`；倒计时 **A（剩余）+ C（最后 10s 变红）**，显示在按钮内 | +| 隐私政策 URL | **GitHub Pages**（仓库站点，如 `…/privacy`） | +| 自动开会话开关 | **先不加** | + +### 7.2 批次 A · App Store 合规（上架前必做） + +- [x] **A1** 隐私政策页面（GitHub Pages `privacy.html` / `docs/privacy`），en + zh-Hans +- [x] **A2** App 内入口：设置页 + Onboarding 底部「隐私政策」链接 +- [ ] **A3** App Store Connect「App 隐私」问卷与政策一致 +- [ ] **A4** 复核 / 更新 `PrivacyInfo.xcprivacy`（主 App + 键盘扩展；麦克风、UserDefaults 等） +- [x] **A5** 更新 `NSMicrophoneUsageDescription` / `NSSpeechRecognitionUsageDescription`（含 Flow 自动会话说明） +- [x] **A6** 「完全访问」专项说明（Onboarding + 设置：用途、不上传击键） +- [ ] **A7** 云端模式披露：润色时仅文字发往用户配置的 API +- [ ] **A8**（可选）支持邮箱 / 用户协议 + +### 7.3 批次 B · 分步权限引导 + +- [x] **B1** Onboarding 麦克风页（说明 + 按钮触发系统弹窗） +- [x] **B2** Onboarding 语音识别页 +- [x] **B3** 键盘 + 「允许完全访问」图文 + 跳转系统设置 +- [x] **B4** 与 `PermissionPrimer` 合并；仅首次 / 权限未定时展示 +- [x] **B5** 权限被拒降级页（去设置） + +### 7.4 批次 C · 语音会话自动化 + +- [x] **C1** 进 App + 权限 OK → 自动 `startSession`（`OSGKeyboardApp` / Home） +- [x] **C2** 冷启动 `checkExistingSession`（`FlowSessionManager` init） +- [x] **C3** Home 卡片 UX：进行中 + 结束；失败原因 + 去设置；隐藏手动「启动」 +- [x] **C4** 与 `scenePhase.active` 续期对齐 + +### 7.5 批次 D · 键盘点按录音 + 60s 倒计时 + +- [x] **D1** `RecordButton` 改为 toggle（替换长按手势） +- [x] **D2** 识别中禁用按钮 +- [x] **D3** 按钮内剩余时间倒计时（`M:SS`） +- [x] **D4** 最后 10 秒变红/橙（A+C） +- [x] **D5** 60s 到点自动 `stopped` → 「识别中…」 +- [x] **D6** 无障碍 / 占位文案改为「点按说话」类 + +### 7.6 批次 E · 多语言完善 + +- [ ] **E1** 键盘扩展：移除 `KeyboardL10n` 硬编码，统一 `Localizable.strings` +- [ ] **E2** `KeyboardViewController` 硬编码中文迁入 strings +- [ ] **E3** 批次 B/C/D/A 新增文案 en + zh-Hans 成对 + +### 7.7 批次 F · Phase 3 收尾 + +- [ ] **F1** B3：会话过期键盘提示、Darwin（可选）、路由（可选） +- [ ] **F2** B4：全场景回归（含自动开、60s、杀 App 恢复） +- [ ] **F3** 更新 §5 验收勾选 + +### 7.8 任务追踪（Phase 4） + +- [x] P4-0：产品规格拍板（交互、权限、自动会话、60s、隐私 URL） +- [ ] P4-A：上架合规批次（A1/A2/A5 代码侧已完成；A3/A4 待人工） +- [x] P4-B：权限引导 +- [x] P4-C：会话自动化 +- [x] P4-D：键盘点按 + 倒计时 +- [x] P4-E：多语言（核心文案；KeyboardL10n 硬编码待全量迁移） +- [ ] P4-F：Phase 3 收尾 + B4 回归 + +--- + +## 4) 完成记录 + +| 日期 | 任务ID | 变更摘要 | 状态 | +|---|---|---|---| +| 2026-06-19 | A1-A4 | 架构研究、方案选择、蓝图与追踪文档 | Done | +| 2026-06-19 | B1-B2 | Flow IPC + 键盘链路；修正 continuous capture 音频层 | Done | +| 2026-06-19 | B4-partial | 真机：本地 + 在线 Flow 可用 | Done | +| 2026-06-19 | P4-0 | Phase 4 产品规格与合规任务清单拍板 | Done | + diff --git a/docs/.nojekyll b/docs/.nojekyll new file mode 100644 index 0000000..e69de29 diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..d0c481e --- /dev/null +++ b/docs/index.html @@ -0,0 +1,149 @@ + + + + + + OSGKeyboard + + + + +
+
+

OSGKeyboard

+

Tap to talk. Polished text in any app.

+

中文

+
+ +
+

Voice dictation keyboard for iOS

+

+ OSGKeyboard is a custom keyboard extension that transcribes your voice on-device + and optionally polishes the result through an LLM you configure. + Works in WeChat, Notes, Mail, ChatGPT, and anywhere a keyboard appears. +

+
+ View on GitHub + Privacy Policy +
+
+ +
+

Privacy at a glance

+

+ Audio is processed on your device for transcription. We do not log ordinary keystrokes. + In Cloud polish mode, only the transcribed text (not audio) is sent to your chosen API provider. +

+
+ Read full policy +
+
+ + + +
+ +
+

OSGKeyboard

+

点按说话，任意 App 里获得润色文字。

+
+ +
+

iOS 语音输入键盘

+

+ OSGKeyboard 是一款自定义键盘扩展：在设备端转写语音，并可选通过你配置的 LLM 润色结果。 + 适用于微信、备忘录、邮件、ChatGPT 等所有出现键盘的场景。 +

+
+ GitHub 仓库 + 隐私政策 +
+
+ +
+

隐私摘要

+

+ 音频在设备端转写。我们不会记录普通击键内容。 + 云端润色模式下，仅发送转写文字（非音频）到你选择的 API 服务商。 +

+
+ 查看完整政策 +
+
+ + +
+ + diff --git a/docs/privacy/index.html b/docs/privacy/index.html new file mode 100644 index 0000000..f6c28bc --- /dev/null +++ b/docs/privacy/index.html @@ -0,0 +1,88 @@ + + + + + + OSGKeyboard Privacy Policy + + + +

中文

+

OSGKeyboard Privacy Policy

+

Last updated: June 19, 2026

+

OSGKeyboard is a custom iOS keyboard that turns your voice into text. This policy explains what data the app processes and how it is used.

+ +

What we collect

+
    +
  • Voice audio — captured only while you actively record. On-device mode transcribes locally with Apple’s speech APIs; raw audio is not uploaded by OSGKeyboard.
    • +
  • Transcribed text — in Cloud polish mode, the final text (not audio) may be sent to the LLM provider you configure (e.g. OpenAI) for punctuation and formatting.
    • +
  • API credentials — stored in the iOS Keychain on your device and shared only between the main app and keyboard extension via an App Group.
    • +
  • App preferences — engine mode, language, and keyboard settings stored in App Group UserDefaults on your device.
    • +
+ +

What we do not collect

+
    +
  • We do not log or upload ordinary keystrokes you type with the keyboard.
    • +
  • We do not operate analytics or advertising SDKs.
    • +
  • We do not sell personal data.
    • +
+ +

Permissions

+
    +
  • Microphone — required for voice input and background voice sessions.
    • +
  • Speech recognition — required for on-device transcription.
    • +
  • Full Access — required so the keyboard can reach the microphone, read your API key, and communicate with the main app. Full Access does not grant us access to everything you type; we do not exfiltrate keystrokes.
    • +
+ +

Third parties

+

When you choose Cloud polish mode, transcribed text is sent to the API endpoint you configure. That provider’s privacy policy applies to those requests.

+ +

Data retention

+

Settings and API keys remain on your device until you delete the app or reset settings. Transcription results are passed to the host app you are typing in and are not stored long-term by OSGKeyboard.

+ +

Contact

+

Questions: open an issue at github.com/hkgood/OSGKeyboard.

+ +
+

OSGKeyboard 隐私政策

+

更新日期：2026 年 6 月 19 日

+

OSGKeyboard 是一款 iOS 自定义键盘，可将语音转为文字。本政策说明应用处理哪些数据及用途。

+ +

我们处理的数据

+
    +
  • 语音音频 — 仅在你主动录音时采集。本地模式在设备端通过 Apple 语音识别转写，OSGKeyboard 不会上传原始录音。
    • +
  • 转写文字 — 云端润色模式下，最终文字（非音频）可能发送到你配置的 LLM 服务商以整理标点和格式。
    • +
  • API 凭证 — 保存在设备 Keychain，仅通过 App Group 在主 App 与键盘扩展间共享。
    • +
  • 应用偏好 — 引擎、语言等设置保存在设备 App Group 中。
    • +
+ +

我们不收集的内容

+
    +
  • 我们不会记录或上传你平时在键盘上的击键内容。
    • +
  • 我们不会集成广告或第三方分析 SDK。
    • +
  • 我们不会出售个人数据。
    • +
+ +

权限说明

+
    +
  • 麦克风 — 语音输入与后台语音会话所需。
    • +
  • 语音识别 — 端侧转写所需。
    • +
  • 完全访问 — 使键盘能使用麦克风、读取 API Key 并与主 App 通信。完全访问不代表我们会收集全部击键内容。
    • +
+ +

第三方

+

选择云端润色时，转写文字会发往你配置的 API，该服务商的隐私政策适用于相关请求。

+ +

数据保留

+

设置与 API Key 保留在设备上，直至卸载或重置。识别结果写入你正在使用的宿主 App，OSGKeyboard 不会长期存储。

+ +

联系

+

问题反馈：github.com/hkgood/OSGKeyboard

+ + diff --git a/project.yml b/project.yml index 9973f0b..c34423f 100644 --- a/project.yml +++ b/project.yml @@ -55,8 +55,8 @@ targets: # each request. The first entry below becomes each process's # default access group, so the Keychain helper does not need to # specify kSecAttrAccessGroup explicitly. - com.apple.security.keychain-access-groups: - - com.osgkeyboard.shared + keychain-access-groups: + - $(AppIdentifierPrefix)com.osgkeyboard.shared resources: - path: OSGKeyboard/Assets.xcassets - path: OSGKeyboard/en.lproj @@ -80,8 +80,10 @@ targets: - UIInterfaceOrientationPortrait UIApplicationSceneManifest: UIApplicationSupportsMultipleScenes: false - NSMicrophoneUsageDescription: "OSGKeyboard needs microphone access to transcribe your voice into text." - NSSpeechRecognitionUsageDescription: "OSGKeyboard uses on-device speech recognition to transcribe your dictation. Audio never leaves your device." + NSMicrophoneUsageDescription: "OSGKeyboard uses the microphone for voice dictation and keeps a background audio session active while a voice session is running." + NSSpeechRecognitionUsageDescription: "OSGKeyboard uses on-device speech recognition to transcribe your voice. Audio is processed on your device and is not uploaded for transcription." + UIBackgroundModes: + - audio NSAppTransportSecurity: NSAllowsArbitraryLoads: false CFBundleURLTypes: @@ -111,8 +113,9 @@ targets: excludes: - "en.lproj" - "zh-Hans.lproj" - - path: OSGKeyboardExt/en.lproj - - path: OSGKeyboardExt/zh-Hans.lproj + resources: + - path: OSGKeyboardExt/en.lproj/Localizable.strings + - path: OSGKeyboardExt/zh-Hans.lproj/Localizable.strings settings: base: IPHONEOS_DEPLOYMENT_TARGET: "26.0" @@ -124,16 +127,16 @@ targets: # Shared with the host app so the keyboard extension can read the # user's LLM API key. See OSGKeyboard/OSGKeyboard.entitlements # for the full rationale. - com.apple.security.keychain-access-groups: - - com.osgkeyboard.shared + keychain-access-groups: + - $(AppIdentifierPrefix)com.osgkeyboard.shared info: path: OSGKeyboardExt/Info.plist properties: CFBundleDisplayName: OSGKeyboard CFBundleShortVersionString: "$(MARKETING_VERSION)" CFBundleVersion: "$(CURRENT_PROJECT_VERSION)" - NSMicrophoneUsageDescription: "OSGKeyboard needs microphone access to transcribe your voice into text." - NSSpeechRecognitionUsageDescription: "OSGKeyboard uses on-device speech recognition to transcribe your dictation. Audio never leaves your device." + NSMicrophoneUsageDescription: "OSGKeyboard uses the microphone for voice dictation and keeps a background audio session active while a voice session is running." + NSSpeechRecognitionUsageDescription: "OSGKeyboard uses on-device speech recognition to transcribe your voice. Audio is processed on your device and is not uploaded for transcription." NSExtension: NSExtensionAttributes: IsASCIICapable: false