feat: TypeWhisper Flow sessions, Phase 4 UX, and GitHub Pages privacy site

Migrate keyboard dictation to continuous Flow sessions with auto-start,
tap-to-toggle recording, 60s countdown, five-step onboarding, and App Group
IPC. Add docs/ GitHub Pages site with en/zh privacy policy for App Store compliance.
This commit is contained in:
Rocky
2026-06-19 18:05:50 +08:00
parent 6148d05093
commit 7f059dbd45
48 changed files with 3815 additions and 1065 deletions
+3
View File
@@ -50,3 +50,6 @@ Signing.local.xcconfig
# Generated by XcodeGen
*.xcodeproj/
# Local DerivedData (when using -derivedDataPath in repo)
.derivedData/
Binary file not shown.

Before

Width:  |  Height:  |  Size: 44 KiB

After

Width:  |  Height:  |  Size: 39 KiB

+6 -2
View File
@@ -42,14 +42,18 @@
<false/>
</dict>
<key>NSMicrophoneUsageDescription</key>
<string>OSGKeyboard needs microphone access to transcribe your voice into text.</string>
<string>OSGKeyboard uses the microphone for voice dictation and keeps a background audio session active while a voice session is running.</string>
<key>NSSpeechRecognitionUsageDescription</key>
<string>OSGKeyboard uses on-device speech recognition to transcribe your dictation. Audio never leaves your device.</string>
<string>OSGKeyboard uses on-device speech recognition to transcribe your voice. Audio is processed on your device and is not uploaded for transcription.</string>
<key>UIApplicationSceneManifest</key>
<dict>
<key>UIApplicationSupportsMultipleScenes</key>
<false/>
</dict>
<key>UIBackgroundModes</key>
<array>
<string>audio</string>
</array>
<key>UILaunchScreen</key>
<dict>
<key>UIColorName</key>
+2 -2
View File
@@ -8,9 +8,9 @@
</array>
<key>com.apple.security.device.audio-input</key>
<true/>
<key>com.apple.security.keychain-access-groups</key>
<key>keychain-access-groups</key>
<array>
<string>com.osgkeyboard.shared</string>
<string>$(AppIdentifierPrefix)com.osgkeyboard.shared</string>
</array>
</dict>
</plist>
+33 -2
View File
@@ -6,14 +6,19 @@ import OSGKeyboardShared
@main
struct OSGKeyboardApp: App {
@Environment(\.scenePhase) private var scenePhase
@StateObject private var config = ProviderConfig.shared
@StateObject private var dictationCoordinator = DictationSessionCoordinator()
@StateObject private var flowManager = FlowSessionManager()
var body: some Scene {
WindowGroup {
ThemedRoot {
if AppGroup.isAvailable {
if config.isConfigured {
if config.hasCompletedOnboarding {
HomeView()
.onAppear { flowManager.autoStartIfNeeded() }
} else {
OnboardingView(config: config)
}
@@ -21,6 +26,32 @@ struct OSGKeyboardApp: App {
AppGroupErrorView()
}
}
.environmentObject(flowManager)
.onOpenURL { url in
guard url.scheme == "osgkeyboard" else { return }
switch url.host {
case "dictate":
dictationCoordinator.present()
case "startflow":
flowManager.startSession()
default:
break
}
}
.fullScreenCover(isPresented: $dictationCoordinator.isPresenting) {
DictationCaptureView(
config: config,
coordinator: dictationCoordinator
)
}
.onChange(of: scenePhase) { _, phase in
guard phase == .active, AppGroup.isAvailable, config.hasCompletedOnboarding else { return }
if flowManager.isActive {
flowManager.extendSession()
} else {
flowManager.autoStartIfNeeded()
}
}
}
}
}
}
+79
View File
@@ -0,0 +1,79 @@
// AppPermissions.swift
// OSGKeyboard · Main App
//
// Central permission status for onboarding and Flow session startup.
import AVFoundation
import Speech
import UIKit
enum AppPermissions {
enum MicStatus: Equatable {
case undetermined
case granted
case denied
}
enum SpeechStatus: Equatable {
case undetermined
case granted
case denied
case restricted
}
static var micStatus: MicStatus {
switch AVAudioApplication.shared.recordPermission {
case .granted: return .granted
case .denied: return .denied
case .undetermined: return .undetermined
@unknown default: return .denied
}
}
static var speechStatus: SpeechStatus {
switch SFSpeechRecognizer.authorizationStatus() {
case .authorized: return .granted
case .denied: return .denied
case .restricted: return .restricted
case .notDetermined: return .undetermined
@unknown default: return .denied
}
}
/// Both permissions required for Flow voice sessions.
static var flowRequirementsMet: Bool {
micStatus == .granted && speechStatus == .granted
}
/// Show guided permission pages when any Flow permission is not granted.
static var needsPermissionGuidance: Bool {
micStatus != .granted || speechStatus != .granted
}
static func requestMicrophone() async -> Bool {
switch micStatus {
case .granted: return true
case .denied: return false
case .undetermined: return await AVAudioApplication.requestRecordPermission()
}
}
static func requestSpeechRecognition() async -> Bool {
switch speechStatus {
case .granted: return true
case .denied, .restricted: return false
case .undetermined:
return await withCheckedContinuation { (cont: CheckedContinuation<Bool, Never>) in
SFSpeechRecognizer.requestAuthorization { status in
cont.resume(returning: status == .authorized)
}
}
}
}
static func openSystemSettings() {
guard let url = URL(string: UIApplication.openSettingsURLString) else { return }
UIApplication.shared.open(url)
}
}
@@ -0,0 +1,396 @@
// FlowSessionManager.swift
// OSGKeyboard · Main App
//
// Session Owner for TypeWhisper-style Flow dictation: continuous
// `.playAndRecord` capture for the whole session, utterance gating for
// ASR, optional LLM polish, and App Group result delivery.
import Foundation
import AVFoundation
import Speech
import OSGKeyboardShared
@MainActor
final class FlowSessionManager: ObservableObject {
@Published private(set) var isActive = false
@Published private(set) var isStarting = false
@Published private(set) var sessionExpiresAt: Date?
/// Non-nil when continuous capture failed or permissions are missing.
@Published private(set) var sessionWarning: String?
private let capture = FlowContinuousCapture()
private let asr: ASRService = ASRServiceFactory.make()
private let polisher = PolishingService()
private let store = AppGroupStore()
private var pollingTask: Task<Void, Never>?
private var heartbeatTask: Task<Void, Never>?
private var expiryTask: Task<Void, Never>?
private var levelTask: Task<Void, Never>?
private var startTask: Task<Void, Never>?
private var isUtteranceRecording = false
private var finalizeTask: Task<Void, Never>?
private var asrTask: Task<Void, Never>?
private var currentPartial = ""
private var lastFinal = ""
init() {
Task { @MainActor [weak self] in
await self?.bootstrapFromStorageIfNeeded()
}
}
// MARK: - Public
/// Starts a Flow session: permissions continuous capture App Group active.
func startSession(duration: TimeInterval = FlowSessionKeys.defaultSessionDuration) {
guard AppGroup.isAvailable else {
debug("cannot start flow session: App Group unavailable")
return
}
if isActive {
extendSession(duration: duration)
return
}
startTask?.cancel()
startTask = Task { @MainActor [weak self] in
await self?.startSessionAsync(duration: duration)
}
}
/// Called on launch / foreground when onboarding is complete.
func autoStartIfNeeded() {
guard AppGroup.isAvailable else { return }
guard !isActive, !isStarting else { return }
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
return
}
let markedActive = AppGroup.defaults.bool(forKey: FlowSessionKeys.flowSessionActive)
if markedActive, FlowSessionBridge.remainingSessionDuration() != nil {
Task { await bootstrapFromStorageIfNeeded() }
return
}
startSession()
}
/// Reattach capture when the host app was killed but the session has not expired.
func bootstrapFromStorageIfNeeded() async {
guard AppGroup.isAvailable, !isActive else { return }
let markedActive = AppGroup.defaults.bool(forKey: FlowSessionKeys.flowSessionActive)
guard markedActive, let remaining = FlowSessionBridge.remainingSessionDuration(), remaining > 0 else {
if markedActive {
FlowSessionBridge.markSessionInactive()
}
return
}
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
return
}
isStarting = true
sessionWarning = nil
defer { isStarting = false }
do {
try capture.start()
} catch {
let message = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
sessionWarning = message
FlowSessionBridge.markSessionInactive()
debug("bootstrap capture failed: \(message)")
return
}
FlowSessionBridge.writeHeartbeat()
isActive = true
if let expires = FlowSessionBridge.sessionExpiresAt() {
sessionExpiresAt = Date(timeIntervalSince1970: expires)
}
startHeartbeat()
startPolling()
startLevelPublishing()
scheduleExpiry(after: remaining)
debug("Flow session restored (\(Int(remaining))s remaining)")
}
func endSession() {
guard isActive else { return }
debug("Flow session ended")
startTask?.cancel()
startTask = nil
pollingTask?.cancel()
pollingTask = nil
heartbeatTask?.cancel()
heartbeatTask = nil
expiryTask?.cancel()
expiryTask = nil
levelTask?.cancel()
levelTask = nil
finalizeTask?.cancel()
finalizeTask = nil
asrTask?.cancel()
asrTask = nil
if isUtteranceRecording {
capture.cancelUtterance()
asr.cancel()
isUtteranceRecording = false
}
capture.stop()
FlowSessionBridge.markSessionInactive()
isActive = false
sessionExpiresAt = nil
sessionWarning = nil
currentPartial = ""
lastFinal = ""
}
func extendSession(duration: TimeInterval = FlowSessionKeys.defaultSessionDuration) {
FlowSessionBridge.extendSession(by: duration)
sessionExpiresAt = Date().addingTimeInterval(duration)
scheduleExpiry(after: duration)
}
// MARK: - Session start
private func startSessionAsync(duration: TimeInterval) async {
isStarting = true
sessionWarning = nil
defer { isStarting = false }
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
return
}
do {
try capture.start()
} catch {
let message = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
sessionWarning = message
debug("continuous capture failed: \(message)")
return
}
FlowSessionBridge.markSessionActive(duration: duration)
isActive = true
sessionExpiresAt = Date().addingTimeInterval(duration)
startHeartbeat()
startPolling()
startLevelPublishing()
scheduleExpiry(after: duration)
debug("Flow session started (\(Int(duration))s), continuous capture running")
}
private func permissionWarningMessage() -> String {
if AppPermissions.micStatus != .granted {
return NSLocalizedString("flow.error.micRequired", comment: "")
}
return NSLocalizedString("flow.error.speechRequired", comment: "")
}
// MARK: - Polling
private func startPolling() {
pollingTask?.cancel()
pollingTask = Task { @MainActor [weak self] in
while !Task.isCancelled {
self?.handleKeyboardSignal()
try? await Task.sleep(nanoseconds: 100_000_000)
}
}
}
private func handleKeyboardSignal() {
switch FlowSessionBridge.recordingState() {
case .recording:
guard !isUtteranceRecording else { return }
beginUtterance()
case .stopped:
guard isUtteranceRecording else { return }
endUtterance()
case .aborted:
abortUtterance()
case .idle, .processing:
break
}
}
private func beginUtterance() {
guard capture.running else {
failUtterance(message: NSLocalizedString("flow.error.audioUnavailable", comment: ""))
return
}
finalizeTask?.cancel()
asrTask?.cancel()
asr.cancel()
currentPartial = ""
lastFinal = ""
let localeId = store.localeId
FlowSessionBridge.setTranscriptionLanguage(localeId)
FlowSessionBridge.clearPendingTranscription()
let locale = SpeechLocaleResolver.resolve(localeId)
let stream = capture.beginUtterance()
let events = asr.transcribe(stream: stream, locale: locale)
isUtteranceRecording = true
asrTask = Task { @MainActor [weak self] in
guard let self else { return }
for await event in events {
switch event {
case .capability:
break
case .partial(let text):
self.currentPartial = text
case .final(let text):
self.lastFinal = text.trimmingCharacters(in: .whitespacesAndNewlines)
self.currentPartial = ""
case .error(let message):
self.debug("asr error: \(message)")
if self.isUtteranceRecording {
self.failUtterance(message: message)
}
}
}
}
debug("utterance recording started")
}
private func endUtterance() {
isUtteranceRecording = false
capture.endUtterance()
FlowSessionBridge.setRecordingState(.processing)
finalizeTask?.cancel()
finalizeTask = Task { @MainActor [weak self] in
await self?.finalizeUtterance()
}
debug("utterance stopped, finalizing")
}
private func abortUtterance() {
isUtteranceRecording = false
finalizeTask?.cancel()
asrTask?.cancel()
asr.cancel()
capture.cancelUtterance()
currentPartial = ""
lastFinal = ""
FlowSessionBridge.setRecordingState(.idle)
debug("utterance aborted")
}
private func failUtterance(message: String) {
isUtteranceRecording = false
asrTask?.cancel()
asr.cancel()
capture.cancelUtterance()
currentPartial = ""
lastFinal = ""
FlowSessionBridge.storeTranscriptionError(message)
debug("utterance failed: \(message)")
}
private func finalizeUtterance() async {
let deadline = Date().addingTimeInterval(30)
while Date() < deadline {
if !lastFinal.isEmpty { break }
try? await Task.sleep(nanoseconds: 100_000_000)
}
var text = lastFinal.trimmingCharacters(in: .whitespacesAndNewlines)
if text.isEmpty {
text = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
}
guard !text.isEmpty else {
FlowSessionBridge.storeTranscriptionError(
NSLocalizedString("flow.error.noSpeech", comment: "")
)
return
}
let engineMode = store.engineMode
let modeId = store.modeId
let shouldPolish = engineMode != "local" && modeId == "polish"
if shouldPolish {
do {
let polished = try await polisher.polish(text)
FlowSessionBridge.storeTranscriptionResult(polished)
} catch {
FlowSessionBridge.storeTranscriptionResult(text)
}
} else {
FlowSessionBridge.storeTranscriptionResult(text)
}
currentPartial = ""
lastFinal = ""
debug("utterance finalized length=\(text.count)")
}
// MARK: - Level publishing (main thread only)
private func startLevelPublishing() {
levelTask?.cancel()
levelTask = Task { @MainActor [weak self] in
while !Task.isCancelled {
guard let self, self.isActive else { break }
let levels = self.capture.currentAudioLevels()
if levels.contains(where: { $0 > 0 }) {
FlowSessionBridge.storeAudioLevels(levels)
}
try? await Task.sleep(nanoseconds: 100_000_000)
}
}
}
// MARK: - Timers
private func startHeartbeat() {
heartbeatTask?.cancel()
FlowSessionBridge.writeHeartbeat()
heartbeatTask = Task { @MainActor [weak self] in
while !Task.isCancelled {
FlowSessionBridge.writeHeartbeat()
try? await Task.sleep(nanoseconds: 1_000_000_000)
guard self?.isActive == true else { break }
}
}
}
private func scheduleExpiry(after duration: TimeInterval) {
expiryTask?.cancel()
expiryTask = Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: UInt64(duration * 1_000_000_000))
guard !Task.isCancelled else { return }
self?.endSession()
}
}
private func debug(_ message: String) {
#if DEBUG
print("🌊[FlowSession] \(message)")
#endif
}
}
+11
View File
@@ -0,0 +1,11 @@
// LegalLinks.swift
// OSGKeyboard · Main App
import Foundation
enum LegalLinks {
/// Public privacy policy (GitHub Pages).
static var privacyPolicyURL: URL? {
URL(string: "https://hkgood.github.io/OSGKeyboard/privacy/")
}
}
@@ -0,0 +1,15 @@
// PermissionPrimer.swift
// OSGKeyboard · Main App
//
// Legacy entry point permission prompts now run inside Onboarding.
// Kept as a no-op so older call sites compile without re-firing TCC
// dialogs on every launch.
import Foundation
@MainActor
enum PermissionPrimer {
static func primeIfNeeded() async {
// Permissions are requested step-by-step in OnboardingView.
}
}
@@ -0,0 +1,146 @@
// DictationCaptureView.swift
// OSGKeyboard · Main App
//
// Host-app recording surface for keyboard handoff.
// Uses the shared `LiveDictationController` the same entry point as
// the keyboard preview sheet.
import SwiftUI
import OSGKeyboardShared
@MainActor
final class DictationSessionCoordinator: ObservableObject {
@Published var isPresenting: Bool = false
func present() {
isPresenting = true
}
func dismiss() {
isPresenting = false
}
}
struct DictationCaptureView: View {
@Environment(\.dismiss) private var dismiss
@Environment(\.themePalette) private var palette: ThemePalette
@ObservedObject var config: ProviderConfig
@ObservedObject var coordinator: DictationSessionCoordinator
@StateObject private var dictation = LiveDictationController()
@State private var statusText: String = "准备录音..."
@State private var isSaving: Bool = false
var body: some View {
ZStack {
palette.background.ignoresSafeArea()
VStack(spacing: Spacing.lg) {
Spacer()
Image(systemName: "mic.circle.fill")
.font(.system(size: 84, weight: .light))
.foregroundStyle(palette.accent)
Text(titleText)
.font(TypeStyle.title2)
.foregroundStyle(palette.textPrimary)
Text(statusText)
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg)
ProgressView(value: dictation.level, total: 1.0)
.tint(palette.accent)
.padding(.horizontal, Spacing.lg)
Spacer()
HStack(spacing: Spacing.sm) {
Button {
cancelAndClose()
} label: {
Text("取消")
.secondaryButton()
}
.buttonStyle(.plain)
.disabled(isSaving)
Button {
stopAndFinalize()
} label: {
Text("完成")
.primaryButton()
}
.buttonStyle(.plain)
.disabled(isSaving || dictation.phase != .recording)
}
.padding(.horizontal, Spacing.md)
.padding(.bottom, Spacing.lg)
}
}
.onAppear {
DictationBridge.setStatus(.requested)
startRecording()
}
.onDisappear {
dictation.stop()
}
.onChange(of: dictation.phase) { _, new in
switch new {
case .recording:
DictationBridge.setStatus(.recording)
statusText = config.isLocalEngine ? "正在实时识别..." : "正在听..."
case .processing:
DictationBridge.setStatus(.transcribing)
statusText = "处理中..."
case .requestingPermission:
statusText = "请求权限中..."
case .denied(let message):
DictationBridge.setStatus(.error, message: message)
statusText = message
case .error(let message):
DictationBridge.setStatus(.error, message: message)
statusText = message
case .idle:
break
}
}
.onChange(of: dictation.currentPartial) { _, new in
guard config.isLocalEngine, !new.isEmpty else { return }
statusText = new
}
.onChange(of: dictation.lastFinal) { _, new in
guard !new.isEmpty else { return }
saveAndClose(new)
}
}
private var titleText: String {
isSaving ? "保存中..." : "语音输入"
}
private func startRecording() {
Task { await dictation.start(localeId: config.localeId) }
}
private func stopAndFinalize() {
dictation.stop()
statusText = "等待识别结果..."
}
private func cancelAndClose() {
dictation.stop()
DictationBridge.setStatus(.cancelled)
coordinator.dismiss()
dismiss()
}
private func saveAndClose(_ transcript: String) {
guard !isSaving else { return }
isSaving = true
DictationBridge.storePendingTranscript(transcript)
coordinator.dismiss()
dismiss()
}
}
+75
View File
@@ -12,6 +12,7 @@ struct HomeView: View {
@Environment(\.themePalette) private var palette: ThemePalette
@ObservedObject var config = ProviderConfig.shared
@EnvironmentObject private var flowManager: FlowSessionManager
@State private var showSettings = false
@State private var showKeyboardPreview = false
@@ -21,6 +22,9 @@ struct HomeView: View {
VStack(spacing: 0) {
statusHeader
.padding(.top, Spacing.xl)
flowSessionCard
.padding(.horizontal, Spacing.md)
.padding(.top, Spacing.md)
Spacer()
heroButton
Spacer()
@@ -37,6 +41,77 @@ struct HomeView: View {
}
}
// MARK: - Flow session
private var flowSessionCard: some View {
VStack(alignment: .leading, spacing: Spacing.xs) {
HStack {
Circle()
.fill(flowStatusColor)
.frame(width: 8, height: 8)
Text(flowStatusTitle)
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
Spacer()
if flowManager.isActive, let expires = flowManager.sessionExpiresAt {
Text(expires, style: .timer)
.font(TypeStyle.caption2)
.foregroundStyle(palette.textTertiary)
.monospacedDigit()
}
}
Text("home.flow.hint")
.font(TypeStyle.caption2)
.foregroundStyle(palette.textTertiary)
.fixedSize(horizontal: false, vertical: true)
if let warning = flowManager.sessionWarning {
Text(warning)
.font(TypeStyle.caption2)
.foregroundStyle(palette.warning)
.fixedSize(horizontal: false, vertical: true)
if !AppPermissions.flowRequirementsMet {
Button {
AppPermissions.openSystemSettings()
} label: {
Text("home.flow.openSettings")
.font(TypeStyle.caption)
.foregroundStyle(palette.accent)
}
.buttonStyle(.plain)
}
}
if flowManager.isActive {
Button {
flowManager.endSession()
} label: {
Text("home.flow.end")
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
}
.buttonStyle(.plain)
}
}
.padding(Spacing.sm)
.background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.large, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: Radius.large, style: .continuous)
.stroke(palette.divider, lineWidth: 0.5)
)
}
private var flowStatusColor: Color {
if flowManager.isActive { return palette.success }
if flowManager.isStarting { return palette.accent }
if flowManager.sessionWarning != nil { return palette.warning }
return palette.warning
}
private var flowStatusTitle: LocalizedStringKey {
if flowManager.isActive { return "home.flow.active" }
if flowManager.isStarting { return "home.flow.starting" }
return "home.flow.inactive"
}
// MARK: - Header
private var statusHeader: some View {
+120 -158
View File
@@ -1,82 +1,32 @@
// KeyboardPreviewSheet.swift
// OSGKeyboard · Main App (Debug)
//
// In-app preview of the keyboard extension. Renders a stand-in
// `KeyboardPreviewStub` so the user can see what the real extension
// looks like, AND drives a *real* `PreviewASRController` so tapping
// the disc actually records from the mic and runs the shared ASR
// service (`SpeechAnalyzer` on iOS 26+),
// and lands recognized text in the top textbox. Without the real ASR
// the preview was a static mock "the text never appears" was a
// fair review note.
// In-app preview of the keyboard extension. Drives the shared
// `LiveDictationController` so tapping record exercises the same
// on-device ASR pipeline as host-app dictation handoff.
//
// Lifecycle (local engine = "transcribe" only, no LLM):
// tap start ASR (idempotent re-entry guard)
// ASR emits .partial / .final
// currentPartial updates the transcript line in real time
// tap stop ASR
// lastFinal event lands
// onChange in this view appends to typedText
// controller resets
// Local engine: partial transcripts stream into the text box live.
// Cloud engine: final transcript is appended when recording stops.
import SwiftUI
import OSGKeyboardShared
struct KeyboardPreviewSheet: View {
@Environment(\.themePalette) private var palette: ThemePalette
@Environment(\.dismiss) private var dismiss
@ObservedObject private var config = ProviderConfig.shared
@StateObject private var asr = PreviewASRController()
@StateObject private var dictation = LiveDictationController()
@State private var showSettings = false
/// Accumulates text in the top textbox. The real keyboard extension
/// inserts directly via `textDocumentProxy`; this preview mirrors
/// that in a parallel `@State` so the user can verify the flow.
@State private var typedText: String = ""
/// Cached snapshot of the previous lastFinal so the .onChange
/// doesn't fire on every body re-render only on real changes.
@State private var lastFinalSeen: String = ""
private enum StubPhase { case idle, recording, processing }
/// Visible phase for the stub. Driven by the real ASR controller
/// when the controller is `recording` we show the recording
/// state; otherwise we show `processing` while a final is in
/// flight and `idle` otherwise.
private var stubPhase: KeyboardPreviewStub.Phase {
switch asr.phase {
case .recording: return .recording
case .processing: return .processing
case .idle: return .idle
case .requestingPermission:
return .recording
case .denied, .error:
return .idle
}
}
/// The transcript line the stub shows under the chips. While
/// recording we surface the live ASR partial; otherwise we surface
/// any error or stay quiet.
private var stubTranscript: String {
switch asr.phase {
case .recording:
return asr.currentPartial.isEmpty ? " " : asr.currentPartial
case .error(let m):
return m
case .denied(let m):
return m
default:
return ""
}
}
/// Text in the box when the current dictation session started.
@State private var dictationAnchorText: String = ""
var body: some View {
ZStack {
palette.background.ignoresSafeArea()
VStack(spacing: 0) {
VStack(spacing: Spacing.md) {
VStack(spacing: Spacing.md) {
Text("preview.title")
.font(TypeStyle.title2)
@@ -87,55 +37,62 @@ struct KeyboardPreviewSheet: View {
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg)
mockTextField.padding(.horizontal, Spacing.md)
controls
.padding(.horizontal, Spacing.md)
}
.padding(.top, Spacing.lg)
Spacer(minLength: 0)
keyboardBlock
Spacer()
}
}
.sheet(isPresented: $showSettings) {
SettingsView()
}
.onChange(of: asr.lastFinal) { _, new in
.onChange(of: dictation.currentPartial) { _, partial in
guard config.isLocalEngine, isDictationActive else { return }
applyLiveTranscript(partial)
}
.onChange(of: dictation.lastFinal) { _, new in
guard !new.isEmpty, new != lastFinalSeen else { return }
lastFinalSeen = new
insertRecognizedText(new)
asr.reset()
if config.isLocalEngine {
applyLiveTranscript(new)
dictationAnchorText = typedText
} else {
insertRecognizedText(new)
}
dictation.reset()
}
// Tear down the ASR pipeline when the sheet leaves the screen
// (preview dismissed, app backgrounded mid-recording, etc.).
// Without this, a leftover `asrTask` keeps the AVAudioSession
// active and the mic permission in use after the user has
// moved on. `asr.stop()` is idempotent it no-ops on
// `.idle`/`.denied`/`.error` so it's safe to call here
// even when the disc is not currently recording.
.onDisappear {
asr.stop()
dictation.stop()
}
}
/// Real `TextField` (was a static placeholder HStack before the
/// first fix). The user can both type into it AND see recognized
/// text land in it as the real ASR fires `.final`.
private var mockTextField: some View {
HStack(spacing: Spacing.xs) {
Image(systemName: "text.cursor")
.foregroundStyle(palette.textSecondary)
TextField(LocalizedStringKey("preview.placeholder"), text: $typedText, axis: .vertical)
.lineLimit(1...4)
.textFieldStyle(.plain)
VStack(alignment: .leading, spacing: Spacing.xs) {
HStack {
Image(systemName: "text.cursor")
.foregroundStyle(palette.textSecondary)
Text(statusTitle)
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
Spacer()
if !typedText.isEmpty {
Button {
typedText = ""
} label: {
Image(systemName: "xmark.circle.fill")
.foregroundStyle(palette.textTertiary)
}
.buttonStyle(.plain)
.accessibilityLabel("preview.clear")
}
}
TextEditor(text: $typedText)
.frame(minHeight: 260)
.scrollContentBackground(.hidden)
.foregroundStyle(palette.textPrimary)
.tint(palette.accent)
if !typedText.isEmpty {
Button {
typedText = ""
} label: {
Image(systemName: "xmark.circle.fill")
.foregroundStyle(palette.textTertiary)
}
.buttonStyle(.plain)
.accessibilityLabel("preview.clear")
}
.padding(.horizontal, 2)
}
.padding(Spacing.md)
.background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.medium))
@@ -145,94 +102,99 @@ struct KeyboardPreviewSheet: View {
)
}
private var keyboardBlock: some View {
VStack(spacing: 0) {
Rectangle()
.fill(palette.divider)
.frame(height: 0.5)
KeyboardPreviewStub(
phase: stubPhase,
level: asr.level,
transcript: stubTranscript,
modeId: config.modeId,
localeId: config.localeId,
onTap: cyclePhase,
openSettings: { showSettings = true },
onModeCycle: cycleMode,
onLocaleCycle: cycleLocale
)
private var controls: some View {
VStack(spacing: Spacing.xs) {
HStack(spacing: Spacing.sm) {
Button {
toggleRecording()
} label: {
Label(isRecording ? "停止录音" : "开始录音", systemImage: isRecording ? "stop.circle.fill" : "mic.circle.fill")
.primaryButton()
}
.buttonStyle(.plain)
Button {
showSettings = true
} label: {
Label("设置", systemImage: "gearshape")
.secondaryButton()
}
.buttonStyle(.plain)
}
Text(serviceLabel)
.font(TypeStyle.caption2)
.foregroundStyle(palette.textTertiary)
.frame(maxWidth: .infinity, alignment: .center)
.lineLimit(2)
.multilineTextAlignment(.center)
}
}
/// Tap on the disc. Drives the *real* ASR pipeline the previous
/// mock that just toggled a hardcoded phase is gone.
private func cyclePhase() {
private var serviceLabel: String {
EngineServiceLabel.summary(
engineMode: config.engineMode,
providerId: config.providerId,
model: config.model
)
}
private var isRecording: Bool {
if case .recording = dictation.phase { return true }
return false
}
private var isDictationActive: Bool {
switch dictation.phase {
case .recording, .processing: return true
default: return false
}
}
private var statusTitle: String {
switch dictation.phase {
case .idle: return "输入框"
case .requestingPermission: return "请求权限中..."
case .recording: return config.isLocalEngine ? "正在实时识别..." : "正在录音..."
case .processing: return "识别中..."
case .denied(let message), .error(let message):
return message
}
}
private func toggleRecording() {
withAnimation(Motion.quick) {
switch asr.phase {
switch dictation.phase {
case .idle, .denied, .error:
let locale = resolveLocale(config.localeId)
Task { await asr.start(locale: locale) }
startRecording()
case .recording:
asr.stop()
stopAndFinalize()
case .requestingPermission, .processing:
break
}
}
}
/// Cycle the input mode on tap of the mode chip. The order mirrors
/// the Settings picker (`off` `transcribe` `polish` wrap)
/// so the user sees the same surface in both places.
///
/// Note: when the user picks `off` we *also* stop any in-flight
/// recording leaving the disc mid-recording in an "off" mode
/// would be a confusing state (the user is recording but the
/// keyboard says it won't insert anything).
private func cycleMode() {
let order = ["off", "transcribe", "polish"]
let current = order.firstIndex(of: config.modeId) ?? 0
let next = order[(current + 1) % order.count]
config.modeId = next
if next == "off" && asr.phase == .recording {
asr.stop()
}
private func startRecording() {
dictationAnchorText = typedText
lastFinalSeen = ""
Task { await dictation.start(localeId: config.localeId) }
}
/// Cycle the locale on tap of the locale chip. Same order as
/// `staticLocales` in `SettingsView.swift` so both surfaces stay
/// in sync. When the user picks a new locale we *also* stop any
/// in-flight recording ASR sessions are bound to the locale
/// they were started with, and continuing to feed buffers into a
/// stale session would produce garbage in the next `.final`.
private func cycleLocale() {
let order = ["auto", "zh-Hans", "zh-Hant", "en-US", "ja-JP", "ko-KR"]
let current = order.firstIndex(of: config.localeId) ?? 0
let next = order[(current + 1) % order.count]
config.localeId = next
if asr.phase == .recording {
asr.stop()
}
private func stopAndFinalize() {
dictation.stop()
}
private func resolveLocale(_ id: String) -> Locale {
if id == "auto" { return .current }
return Locale(identifier: id)
private func applyLiveTranscript(_ transcript: String) {
let composed = DictationTextComposer.compose(anchor: dictationAnchorText, live: transcript)
guard composed != typedText else { return }
typedText = composed
}
/// Append the recognized transcript to the textbox, with a leading
/// space when the existing text doesn't already end in whitespace.
/// Matches what `textDocumentProxy.insertText` does for the real
/// keyboard when the user's draft has no trailing whitespace.
private func insertRecognizedText(_ recognized: String) {
let trimmed = recognized.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
if typedText.isEmpty {
typedText = trimmed
} else if typedText.last == " " || typedText.last == "\n" {
typedText += trimmed
} else {
typedText += " " + trimmed
}
typedText = DictationTextComposer.compose(anchor: typedText, live: trimmed)
}
}
+240 -60
View File
@@ -1,32 +1,26 @@
// OnboardingView.swift
// OSGKeyboard · Main App
//
// Three-step onboarding:
//
// 1) Welcome what the app does, in one sentence
// 2) Enable Settings General Keyboards Add Allow Full Access
// 3) Setup pick a provider, paste a key
//
// We deliberately do NOT use `TabView` with `.page` style for the
// pager. That style wraps the content in a `UIPageViewController`,
// and `UIPageViewController` has a long-standing behavior on some iOS
// versions where
// the keyboard-showing layout reflow on a `TextField` focus is
// misread as a horizontal swipe the page jumps back to step 1
// the moment the user starts typing. Replacing the TabView with a
// `ZStack`-based conditional view sidesteps the bug entirely; we
// give up the swipe-to-page gesture, but the Back/Next buttons at
// the bottom (and the page dots) are the canonical onboarding
// affordance and the user is never more than one tap from the next
// page anyway.
//
// Visual style: one large accent surface, generous whitespace, single CTA
// at the bottom. No tipsy animations, no cheerful illustrations every
// pixel is doing one job.
// Five-step onboarding:
// 1) Welcome
// 2) Microphone permission
// 3) Speech recognition permission
// 4) Enable keyboard + Allow Full Access
// 5) Engine / API setup
import SwiftUI
import OSGKeyboardShared
private enum OnboardingPage: Int, CaseIterable {
case welcome = 0
case microphone
case speech
case keyboard
case api
static let count = 5
}
struct OnboardingView: View {
@Environment(\.themePalette) private var palette: ThemePalette
@@ -38,10 +32,12 @@ struct OnboardingView: View {
palette.background.ignoresSafeArea()
VStack(spacing: 0) {
Group {
switch page {
case 0: WelcomePage()
case 1: EnableKeyboardPage()
default: APISetupPage(config: config)
switch OnboardingPage(rawValue: page) ?? .welcome {
case .welcome: WelcomePage()
case .microphone: MicPermissionPage()
case .speech: SpeechPermissionPage()
case .keyboard: EnableKeyboardPage()
case .api: APISetupPage(config: config)
}
}
.frame(maxWidth: .infinity, maxHeight: .infinity)
@@ -59,7 +55,7 @@ struct OnboardingView: View {
private var pageDots: some View {
HStack(spacing: 6) {
ForEach(0..<3, id: \.self) { i in
ForEach(0..<OnboardingPage.count, id: \.self) { i in
Capsule()
.fill(i == page ? palette.accent : Color.white.opacity(0.18))
.frame(width: i == page ? 18 : 6, height: 6)
@@ -68,6 +64,19 @@ struct OnboardingView: View {
}
}
private var isLastPage: Bool { page == OnboardingPage.api.rawValue }
private var canAdvance: Bool {
switch OnboardingPage(rawValue: page) ?? .welcome {
case .welcome, .keyboard, .api:
return page != OnboardingPage.api.rawValue || config.isConfigured
case .microphone:
return AppPermissions.micStatus != .undetermined
case .speech:
return AppPermissions.speechStatus != .undetermined
}
}
@ViewBuilder
private var bottomBar: some View {
HStack(spacing: Spacing.sm) {
@@ -88,31 +97,31 @@ struct OnboardingView: View {
Button {
withAnimation(Motion.soft) {
if page < 2 { page += 1 }
if isLastPage {
config.hasCompletedOnboarding = true
} else {
page += 1
}
}
} label: {
Text(page == 2
? (config.isConfigured
? NSLocalizedString("common.done", comment: "")
: NSLocalizedString("common.continue", comment: ""))
Text(isLastPage
? NSLocalizedString("common.done", comment: "")
: NSLocalizedString("common.next", comment: ""))
.font(TypeStyle.headline)
.frame(maxWidth: .infinity, minHeight: 50)
.background(
(page == 2 && !config.isConfigured) ? palette.surfaceElevated : palette.accent,
canAdvance ? palette.accent : palette.surfaceElevated,
in: RoundedRectangle(cornerRadius: Radius.large, style: .continuous)
)
.foregroundStyle(
(page == 2 && !config.isConfigured) ? palette.textSecondary : palette.textOnAccent
)
.foregroundStyle(canAdvance ? palette.textOnAccent : palette.textSecondary)
}
.buttonStyle(.plain)
.disabled(page == 2 && !config.isConfigured)
.disabled(!canAdvance)
}
}
}
// MARK: - Page 1: Welcome
// MARK: - Welcome
private struct WelcomePage: View {
@Environment(\.themePalette) private var palette: ThemePalette
@@ -141,6 +150,14 @@ private struct WelcomePage: View {
.padding(.horizontal, Spacing.xl)
PrivacyFootnote()
.padding(.top, Spacing.lg)
if let url = LegalLinks.privacyPolicyURL {
Link(destination: url) {
Text("legal.privacyPolicy")
.font(TypeStyle.caption)
.foregroundStyle(palette.accent)
}
.padding(.top, Spacing.xs)
}
Spacer()
}
}
@@ -151,15 +168,9 @@ private struct PrivacyFootnote: View {
var body: some View {
VStack(alignment: .leading, spacing: 8) {
footnoteRow(icon: "lock.fill",
title: "privacy.audio.title",
body: "privacy.audio.body")
footnoteRow(icon: "wifi",
title: "privacy.network.title",
body: "privacy.network.body")
footnoteRow(icon: "keyboard",
title: "privacy.universal.title",
body: "privacy.universal.body")
footnoteRow(icon: "lock.fill", title: "privacy.audio.title", body: "privacy.audio.body")
footnoteRow(icon: "wifi", title: "privacy.network.title", body: "privacy.network.body")
footnoteRow(icon: "keyboard", title: "privacy.universal.title", body: "privacy.universal.body")
}
.padding(Spacing.md)
.background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.large, style: .continuous))
@@ -170,9 +181,6 @@ private struct PrivacyFootnote: View {
.padding(.horizontal, Spacing.md)
}
// `LocalizedStringKey` (not `String`) so the call-site string
// literals are auto-looked-up in Localizable.strings. Passing a
// plain `String` would just print the key.
private func footnoteRow(icon: String, title: LocalizedStringKey, body: LocalizedStringKey) -> some View {
HStack(alignment: .top, spacing: Spacing.xs) {
Image(systemName: icon)
@@ -192,7 +200,180 @@ private struct PrivacyFootnote: View {
}
}
// MARK: - Page 2: Enable keyboard
// MARK: - Permission pages
private struct MicPermissionPage: View {
@Environment(\.themePalette) private var palette: ThemePalette
@State private var status = AppPermissions.micStatus
@State private var isRequesting = false
var body: some View {
PermissionPageLayout(
icon: "mic.fill",
title: "onboarding.permission.mic.title",
detail: "onboarding.permission.mic.body",
status: statusLabel,
statusColor: statusColor,
primaryTitle: primaryButtonTitle,
primaryDisabled: isRequesting || status == .granted,
onPrimary: { Task { await request() } },
secondaryTitle: status == .denied ? "onboarding.permission.openSettings" : nil,
onSecondary: status == .denied ? { AppPermissions.openSystemSettings() } : nil,
deniedHint: status == .denied ? "onboarding.permission.mic.deniedHint" : nil
)
.onAppear { status = AppPermissions.micStatus }
}
private var statusLabel: LocalizedStringKey {
switch status {
case .undetermined: return "onboarding.permission.status.undetermined"
case .granted: return "onboarding.permission.status.granted"
case .denied: return "onboarding.permission.status.denied"
}
}
private var statusColor: Color {
switch status {
case .granted: return palette.success
case .denied: return palette.warning
case .undetermined: return palette.textTertiary
}
}
private var primaryButtonTitle: LocalizedStringKey {
status == .granted ? "onboarding.permission.status.granted" : "onboarding.permission.mic.allow"
}
private func request() async {
isRequesting = true
_ = await AppPermissions.requestMicrophone()
status = AppPermissions.micStatus
isRequesting = false
}
}
private struct SpeechPermissionPage: View {
@Environment(\.themePalette) private var palette: ThemePalette
@State private var isRequesting = false
var body: some View {
PermissionPageLayout(
icon: "waveform.badge.mic",
title: "onboarding.permission.speech.title",
detail: "onboarding.permission.speech.body",
status: statusLabel,
statusColor: statusColor,
primaryTitle: primaryButtonTitle,
primaryDisabled: isRequesting || AppPermissions.speechStatus == .granted,
onPrimary: { Task { await request() } },
secondaryTitle: speechDenied ? "onboarding.permission.openSettings" : nil,
onSecondary: speechDenied ? { AppPermissions.openSystemSettings() } : nil,
deniedHint: speechDenied ? "onboarding.permission.speech.deniedHint" : nil
)
}
private var speechDenied: Bool {
switch AppPermissions.speechStatus {
case .denied, .restricted: return true
default: return false
}
}
private var statusLabel: LocalizedStringKey {
switch AppPermissions.speechStatus {
case .undetermined: return "onboarding.permission.status.undetermined"
case .granted: return "onboarding.permission.status.granted"
case .denied, .restricted: return "onboarding.permission.status.denied"
}
}
private var statusColor: Color {
switch AppPermissions.speechStatus {
case .granted: return palette.success
case .denied, .restricted: return palette.warning
case .undetermined: return palette.textTertiary
}
}
private var primaryButtonTitle: LocalizedStringKey {
AppPermissions.speechStatus == .granted
? "onboarding.permission.status.granted"
: "onboarding.permission.speech.allow"
}
private func request() async {
isRequesting = true
_ = await AppPermissions.requestSpeechRecognition()
isRequesting = false
}
}
private struct PermissionPageLayout: View {
@Environment(\.themePalette) private var palette: ThemePalette
let icon: String
let title: LocalizedStringKey
let detail: LocalizedStringKey
let status: LocalizedStringKey
let statusColor: Color
let primaryTitle: LocalizedStringKey
let primaryDisabled: Bool
let onPrimary: () -> Void
var secondaryTitle: LocalizedStringKey? = nil
var onSecondary: (() -> Void)? = nil
var deniedHint: LocalizedStringKey? = nil
var body: some View {
VStack(spacing: Spacing.lg) {
Spacer()
Image(systemName: icon)
.font(.system(size: 64, weight: .light))
.foregroundStyle(palette.accent)
VStack(spacing: Spacing.sm) {
Text(title)
.font(TypeStyle.title2)
.foregroundStyle(palette.textPrimary)
Text(detail)
.font(TypeStyle.body)
.foregroundStyle(palette.textSecondary)
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg)
}
HStack(spacing: 6) {
Circle().fill(statusColor).frame(width: 8, height: 8)
Text(status)
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
}
if let deniedHint {
Text(deniedHint)
.font(TypeStyle.caption2)
.foregroundStyle(palette.warning)
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg)
}
VStack(spacing: Spacing.xs) {
Button(action: onPrimary) {
Text(primaryTitle)
.primaryButton()
}
.buttonStyle(.plain)
.disabled(primaryDisabled)
if let secondaryTitle, let onSecondary {
Button(action: onSecondary) {
Text(secondaryTitle)
.secondaryButton()
}
.buttonStyle(.plain)
}
}
.padding(.horizontal, Spacing.md)
Spacer()
}
}
}
// MARK: - Enable keyboard
private struct EnableKeyboardPage: View {
@Environment(\.themePalette) private var palette: ThemePalette
@@ -207,6 +388,11 @@ private struct EnableKeyboardPage: View {
Text("onboarding.enable.title")
.font(TypeStyle.title2)
.foregroundStyle(palette.textPrimary)
Text("onboarding.enable.fullAccessNote")
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg)
}
VStack(alignment: .leading, spacing: Spacing.sm) {
step(num: 1, text: NSLocalizedString("onboarding.enable.step1", comment: ""))
@@ -218,13 +404,12 @@ private struct EnableKeyboardPage: View {
.padding(.horizontal, Spacing.md)
Button {
if let url = URL(string: UIApplication.openSettingsURLString) {
UIApplication.shared.open(url)
}
AppPermissions.openSystemSettings()
} label: {
Label(LocalizedStringKey("onboarding.enable.openSettings"), systemImage: "arrow.up.right.square")
.primaryButton()
}
.buttonStyle(.plain)
.padding(.horizontal, Spacing.md)
Spacer()
}
@@ -245,7 +430,7 @@ private struct EnableKeyboardPage: View {
}
}
// MARK: - Page 3: API setup
// MARK: - API setup
private struct APISetupPage: View {
@Environment(\.themePalette) private var palette: ThemePalette
@@ -267,20 +452,15 @@ private struct APISetupPage: View {
.padding(.horizontal, Spacing.md)
.padding(.top, Spacing.lg)
// Same Engine picker as Settings see EnginePickerSection.
EnginePickerSection(config: config)
.padding(.horizontal, Spacing.md)
if config.engineMode == "cloud" {
ProviderPickerSection(config: config)
.padding(.horizontal, Spacing.md)
APISettingsCard(config: config)
.padding(.horizontal, Spacing.md)
} else {
// Local path: no LLM, no API key needed. Show a short
// confirmation so the user understands "no further
// setup required".
VStack(alignment: .leading, spacing: Spacing.xs) {
HStack(spacing: Spacing.sm) {
Image(systemName: "checkmark.seal.fill")
+4 -457
View File
@@ -1,462 +1,9 @@
// PreviewASRController.swift
// OSGKeyboard · Main App (Debug)
// OSGKeyboard · Main App
//
// Self-contained ASR controller for the in-app keyboard preview sheet.
// Owns its own AVAudioEngine + AVAudioSession, downsamples to 16 kHz
// mono Float32, and feeds the `AudioBufferSnapshot` stream to the
// shared `ASRService` (the same pipeline the real keyboard extension
// uses, so the preview exercises the *real* iOS speech APIs, not a
// stub). Without this the in-app preview was a hardcoded transcript
// and "did you actually call SFSpeechRecognizer?" was a fair review
// note.
//
// Why not reuse `AudioCaptureService` from the extension? It lives in
// `OSGKeyboardExt`, an `app-extension` target the main app can't
// import its symbols. We could move it to `OSGKeyboardShared`, but
// `AVAudioSession` lifecycle differs enough between a keyboard
// extension (no background, no recording entitlement surprise) and a
// foreground app that a copy here is the lesser evil.
// Legacy name kept for existing call sites and tests. The implementation
// lives in `OSGKeyboardShared` as `LiveDictationController`.
import Foundation
import AVFoundation
import Speech
import OSGKeyboardShared
@MainActor
final class PreviewASRController: ObservableObject {
enum Phase: Equatable {
case idle
case requestingPermission
case recording
case processing
case denied(String)
case error(String)
}
@Published private(set) var phase: Phase = .idle
/// Normalized 0...1 RMS for the disc level meter. Polled from the
/// audio tap via `Task { @MainActor in ... }` the tap itself
/// runs on a real-time audio thread, so we never touch published
/// state from there.
@Published private(set) var level: Double = 0
@Published private(set) var currentPartial: String = ""
@Published private(set) var errorMessage: String?
/// Set when a `.final` ASR event lands. The owning sheet observes
/// this and appends the text to its textbox, then clears it so the
/// next recording starts from zero.
@Published var lastFinal: String = ""
private let asr: ASRService = ASRServiceFactory.make()
private let audioEngine = AVAudioEngine()
/// `internal` (not `private`) so the regression test in
/// `OSGKeyboardTests/PreviewASRControllerStateTests.swift` can
/// install a known consumer task and assert `stop()` doesn't
/// cancel it. The class is `@MainActor`-isolated, so the
/// natural Swift 6 isolation rules still prevent production
/// code outside the class from racing on it.
var asrTask: Task<Void, Never>?
private var bufferContinuation: AsyncStream<AudioBufferSnapshot>.Continuation?
private var didConfigureAudioSession = false
private var didInstallTap = false
func start(locale: Locale) async {
// Re-entry guard: ignore taps that arrive while we're already
// running. (The sheet's `cyclePhase` is also guarded, but
// async race windows are easier to lock down here.)
switch phase {
case .recording, .requestingPermission, .processing:
return
default:
break
}
// Cancel any leftover consumer task from a previous recording.
// Normally `stop()` lets the task run to completion (so it can
// see the `.final` and transition out of `.processing`), but if
// the user smashed the disc twice stop, then immediately
// start the previous task might still be draining. Cancel it
// here so we don't have two consumer tasks fighting over the
// same `events` stream.
asrTask?.cancel()
asrTask = nil
phase = .requestingPermission
currentPartial = ""
lastFinal = ""
errorMessage = nil
level = 0
// 1. Microphone permission. The helper is `nonisolated` so the
// (iOS < 17) callback closure does not inherit `@MainActor`
// `AVAudioSession.requestRecordPermission` delivers on a TCC
// reply queue, and a `@MainActor`-inferred closure body there
// hits `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift`.
let micGranted = await Self.requestMicrophonePermission()
guard micGranted else {
phase = .denied(NSLocalizedString("keyboard.denied.mic", comment: ""))
return
}
// 2. Speech recognition permission. Same reasoning as above:
// the callback fires on TCC's reply queue, NOT the main queue.
let speechGranted = await Self.requestSpeechRecognitionPermission()
guard speechGranted else {
phase = .denied(NSLocalizedString("keyboard.denied.speech", comment: ""))
return
}
// 3. Audio session only configure once per process.
//
// Category is `.record` (not `.playAndRecord`) because the
// preview never plays back audio it just records from the
// mic and hands the buffers to `SpeechAnalyzer`. On the
// iOS Simulator, `.playAndRecord` requires the
// `AURemoteIO` Audio Unit's *output* side to also be
// enabled, but the simulator's "speaker" reports a 0 Hz
// hardware format, so `AURemoteIO::enable` fails with
// `kAudioUnitErr_FormatNotSupported` (-10851) and any
// subsequent `installTap` traps with "Failed to create tap
// due to format mismatch". `.record` skips the output
// side entirely, so the simulator can record.
//
// The real keyboard extension (`OSGKeyboardExt`) keeps
// `.playAndRecord` because it runs on a real device where
// the output side has a real hardware format, and may want
// to play click sounds / haptic feedback. Only the preview
// needs the simulator-friendly category.
if !didConfigureAudioSession {
do {
let session = AVAudioSession.sharedInstance()
try session.setCategory(.record,
mode: .measurement,
options: [])
try session.setActive(true, options: .notifyOthersOnDeactivation)
didConfigureAudioSession = true
} catch {
phase = .error(String.localizedStringWithFormat(
NSLocalizedString("preview.error.audioSession", comment: ""),
error.localizedDescription
))
return
}
}
// 4. Spin up the engine + ASR.
phase = .recording
startEngineAndASR(locale: locale)
}
func stop() {
// Don't `asrTask?.cancel()` here see the comment in
// `startEngineAndASR` for the full rationale. Short version:
// cancelling the consumer task at the same moment we close the
// audio stream also triggers the producer's
// `continuation.onTermination self?.cancel()` cascade, which
// marks the producer's outer task as cancelled and skips the
// `.final` event. The UI is then left in `.processing` forever
// because no one schedules the transition out. The consumer
// task naturally exits when `events` finishes, so the right
// thing is to let it run.
//
// If a previous `asrTask` is somehow still running (e.g. the
// user smashed the disc twice quickly), `start()` cancels it
// at the entry point as a safety net.
if didInstallTap {
audioEngine.inputNode.removeTap(onBus: 0)
didInstallTap = false
}
if audioEngine.isRunning {
audioEngine.stop()
}
bufferContinuation?.finish()
bufferContinuation = nil
if phase == .recording {
phase = .processing
}
// Deactivate so the user's music resumes if the preview is
// dismissed mid-recording.
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
// Safety net: if the ASR pipeline never produces a `.final`
// (analyzer hang, system glitch, dropped continuation), force
// the UI back to idle after a short delay so the user isn't
// stuck. Normal recordings complete well under a second, so
// the 3-second budget is only hit on the unhappy path; if the
// pipeline finishes first and flips the phase to `.idle` (or
// `.error`), the check below no-ops.
Task { @MainActor [weak self] in
try? await Task.sleep(for: .seconds(3))
guard let self else { return }
if self.phase == .processing {
self.phase = .idle
}
}
}
func reset() {
// Called by the sheet after appending `lastFinal` to the textbox,
// so the next recording can produce a fresh final without us
// double-appending.
lastFinal = ""
if phase == .processing {
phase = .idle
}
}
// MARK: - Engine + ASR
private func startEngineAndASR(locale: Locale) {
let inputNode = audioEngine.inputNode
let hwFormat = inputNode.outputFormat(forBus: 0)
// Pre-flight check: a placeholder / unconfigured input bus
// reports `sampleRate == 0` (or `channelCount == 0`).
// `installTap` on such a bus traps with "Failed to create
// tap due to format mismatch" (an NSException, not a Swift
// `Error`, so we can't `try`/`catch` it). The safest fix
// is to refuse the tap up front and surface a clear
// `.error` phase instead of crashing the app. We've seen
// this on the iOS Simulator when the host's microphone
// permission isn't granted to CoreSimulator, and on
// devices where the audio session is in an unexpected
// state from a previous foreground/background transition.
guard hwFormat.sampleRate > 0, hwFormat.channelCount > 0 else {
phase = .error(
String.localizedStringWithFormat(
NSLocalizedString("preview.error.micUnavailable", comment: ""),
hwFormat.sampleRate,
Int(hwFormat.channelCount)
)
)
return
}
let targetSampleRate: Double = 16_000
guard let targetFormat = AVAudioFormat(
commonFormat: .pcmFormatFloat32,
sampleRate: targetSampleRate,
channels: 1,
interleaved: false
) else {
phase = .error(NSLocalizedString("preview.error.formatCreate", comment: ""))
return
}
guard let converter = AVAudioConverter(from: hwFormat, to: targetFormat) else {
phase = .error(NSLocalizedString("preview.error.converterCreate", comment: ""))
return
}
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
self.bufferContinuation = continuation
// Tap the hardware input. The closure passed to `installTap` runs
// on the AVAudioEngine real-time audio thread. In Swift 6 strict
// concurrency, a closure literal defined inside a `@MainActor`
// method inherits `@MainActor` isolation, which would trip
// `dispatch_assert_queue_fail` on first invocation from the
// audio thread. The fix is to build the actual tap body in a
// `nonisolated` helper (`makeAudioTapBlock`) and have the
// installTap closure be a single function reference function
// references never carry inferred isolation, so the dispatch
// runtime is happy and the body runs wherever AVAudioEngine
// wants it (the audio thread).
let onMeter: @Sendable (Double) -> Void = { [weak self] meter in
Task { @MainActor [weak self] in
guard let self else { return }
// Lightweight smoothing so the disc ring doesn't jitter.
self.level = self.level * 0.55 + meter * 0.45
}
}
let onSnapshot: @Sendable (AudioBufferSnapshot) -> Void = { [weak self] snapshot in
Task { @MainActor [weak self] in
self?.bufferContinuation?.yield(snapshot)
}
}
let tap = Self.makeAudioTapBlock(
converter: converter,
targetFormat: targetFormat,
hwFormat: hwFormat,
targetSampleRate: targetSampleRate,
onMeter: onMeter,
onSnapshot: onSnapshot
)
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
didInstallTap = true
audioEngine.prepare()
do {
try audioEngine.start()
} catch {
phase = .error(String.localizedStringWithFormat(
NSLocalizedString("preview.error.engineStart", comment: ""),
error.localizedDescription
))
return
}
// 5. Wire up ASR.
let events = asr.transcribe(
stream: stream,
locale: locale
)
asrTask = Task { @MainActor [weak self] in
guard let self else { return }
for await event in events {
switch event {
case .capability:
// Could surface on-device vs cloud here; preview
// doesn't need it.
break
case .partial(let s):
self.currentPartial = s
case .final(let s):
let trimmed = s.trimmingCharacters(in: .whitespacesAndNewlines)
self.lastFinal = trimmed
self.currentPartial = ""
if !trimmed.isEmpty {
self.phase = .idle
} else {
// Empty final: nothing recognized. Return to idle
// without triggering a textbox insert.
self.phase = .idle
}
case .error(let m):
self.errorMessage = m
self.phase = .error(m)
}
}
}
}
// MARK: - Permission helpers (nonisolated)
//
// `SFSpeechRecognizer.requestAuthorization` delivers its callback
// on a TCC reply queue, NOT the main queue. If we wrap that
// callback inline in `start(locale:)` which is `@MainActor`
// Swift 6 strict concurrency infers the closure body as
// `@MainActor`, and the runtime crashes on
// `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift` as
// soon as TCC calls us back.
//
// The first attempt (commit `e8a0310`) extracted the entire
// permission request into a `nonisolated static func` helper.
// That worked in isolation, but the Swift 6 optimizer
// inlined those helpers back into `start(locale:)`. After
// inlining, the `withCheckedContinuation` body and the
// `requestAuthorization` callback were re-typed in the
// `@MainActor` context of the caller, and the runtime
// assertion came right back same crash, different symbol:
// `closure #1 in closure #2 in PreviewASRController.start(locale:)`.
//
// The fix that survives inlining is the *function-reference*
// pattern, the same one used for `installTap` in
// `makeAudioTapBlock` below. The callback is built in a
// `nonisolated` static helper that takes a `CheckedContinuation`
// and returns the `(Status) -> Void` handler. The body of that
// helper has no enclosing actor, so the closure is created in
// nonisolated context. When TCC calls us back, the runtime
// sees a nonisolated closure on a non-main queue and is happy.
//
// `cont.resume(...)` is itself thread-safe on
// `CheckedContinuation`, so we don't need to hop back to the
// main actor before resuming.
private nonisolated static func requestMicrophonePermission() async -> Bool {
// iOS 17+ API; the iOS < 17 fallback (`AVAudioSession.recordPermission`
// + `requestRecordPermission` callback) is gone now that the
// deployment target is iOS 26.
switch AVAudioApplication.shared.recordPermission {
case .granted: return true
case .denied: return false
case .undetermined: return await AVAudioApplication.requestRecordPermission()
@unknown default: return false
}
}
private nonisolated static func requestSpeechRecognitionPermission() async -> Bool {
await withCheckedContinuation { (cont: CheckedContinuation<Bool, Never>) in
SFSpeechRecognizer.requestAuthorization(
Self.makeSpeechAuthHandler(continuation: cont)
)
}
}
private nonisolated static func makeSpeechAuthHandler(
continuation: CheckedContinuation<Bool, Never>
) -> @Sendable (SFSpeechRecognizerAuthorizationStatus) -> Void {
return { status in
continuation.resume(returning: status == .authorized)
}
}
// MARK: - Audio tap (nonisolated, runs on AVAudioEngine render thread)
//
// `AVAudioNode.installTap`'s callback fires on the audio engine's
// real-time render thread. In Swift 6 strict concurrency, a closure
// literal defined inside a `@MainActor` method inherits `@MainActor`
// isolation and `dispatch_assert_queue_fail` fires the moment
// the runtime tries to dispatch that closure on a non-main queue.
//
// The trick is to build the actual tap body in a `nonisolated`
// function and have the installTap closure be a *function reference*
// to that helper. Function references never carry inferred
// isolation, so the dispatch runtime is satisfied and the body
// runs wherever AVAudioEngine wants. State updates to
// `self.level` and the AsyncStream continuation hop back to the
// main actor via `Task { @MainActor in }`, which is itself
// safe to call from a non-isolated context.
private nonisolated static func makeAudioTapBlock(
converter: AVAudioConverter,
targetFormat: AVAudioFormat,
hwFormat: AVAudioFormat,
targetSampleRate: Double,
onMeter: @Sendable @escaping (Double) -> Void,
onSnapshot: @Sendable @escaping (AudioBufferSnapshot) -> Void
) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
// `@Sendable` on the returned closure makes the Sendable
// conformance explicit. `AVAudioNodeTapBlock` is declared as
// a plain escaping closure in the SDK; we cast at the call
// site via `as @Sendable`.
return { buffer, _ in
// Downsample + extract samples + compute RMS in one pass.
let ratio = targetSampleRate / hwFormat.sampleRate
let outCapacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio + 0.5)
guard outCapacity > 0,
let converted = AVAudioPCMBuffer(
pcmFormat: targetFormat,
frameCapacity: outCapacity
) else { return }
var error: NSError?
var supplied = false
converter.convert(to: converted, error: &error) { _, outStatus in
if supplied {
outStatus.pointee = .endOfStream
return nil
}
supplied = true
outStatus.pointee = .haveData
return buffer
}
if error != nil { return }
let n = Int(converted.frameLength)
var samples = [Float](repeating: 0, count: n)
var sumSquares: Float = 0
if let channelData = converted.floatChannelData?[0] {
for i in 0..<n {
let v = channelData[i]
samples[i] = v
sumSquares += v * v
}
}
let rms = n > 0 ? sqrtf(sumSquares / Float(n)) : 0
// RMS for speech is typically 0.02-0.2; the 4x gain pushes
// normal speech into the 0.4-0.8 range for the disc meter.
let meter = min(Double(rms) * 4.0, 1.0)
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: targetSampleRate)
onMeter(meter)
onSnapshot(snapshot)
}
}
}
typealias PreviewASRController = LiveDictationController
+34
View File
@@ -33,6 +33,7 @@ struct SettingsView: View {
if config.engineMode == "cloud" {
promptSection
}
privacySection
resetButton
}
.padding(.horizontal, Spacing.md)
@@ -253,6 +254,39 @@ struct SettingsView: View {
}
}
// MARK: - Privacy
private var privacySection: some View {
VStack(alignment: .leading, spacing: Spacing.xs) {
sectionHeader("settings.privacy.title", subtitle: nil)
VStack(alignment: .leading, spacing: Spacing.sm) {
if let url = LegalLinks.privacyPolicyURL {
Link(destination: url) {
HStack {
Text("settings.privacy.policy")
.font(TypeStyle.body)
.foregroundStyle(palette.accent)
Spacer()
Image(systemName: "arrow.up.right")
.font(.system(size: 12, weight: .semibold))
.foregroundStyle(palette.textTertiary)
}
}
}
VStack(alignment: .leading, spacing: 4) {
Text("settings.privacy.fullAccess.title")
.font(TypeStyle.caption)
.foregroundStyle(palette.textPrimary)
Text("settings.privacy.fullAccess.body")
.font(TypeStyle.caption2)
.foregroundStyle(palette.textSecondary)
.fixedSize(horizontal: false, vertical: true)
}
}
.cardSurface()
}
}
// MARK: - Reset
private var resetButton: some View {
+42 -5
View File
@@ -3,7 +3,21 @@
"OSGKeyboard" = "OSGKeyboard";
/* Onboarding */
"onboarding.welcome.subtitle" = "Hold to talk. Release for polished text, in any app.";
"onboarding.welcome.subtitle" = "Tap to talk. Tap again to finish — polished text in any app.";
"onboarding.enable.fullAccessNote" = "Full Access lets the keyboard reach the microphone and your LLM API. We never log what you type.";
"onboarding.permission.mic.title" = "Microphone access";
"onboarding.permission.mic.body" = "Required for voice input. Audio is processed on your device for transcription.";
"onboarding.permission.mic.allow" = "Allow microphone";
"onboarding.permission.mic.deniedHint" = "Microphone was denied. Open Settings to enable it, or tap Next to continue.";
"onboarding.permission.speech.title" = "Speech recognition";
"onboarding.permission.speech.body" = "On-device speech recognition turns your voice into text. Audio never leaves your device.";
"onboarding.permission.speech.allow" = "Allow speech recognition";
"onboarding.permission.speech.deniedHint" = "Speech recognition was denied. Open Settings to enable it, or tap Next to continue.";
"onboarding.permission.openSettings" = "Open Settings";
"onboarding.permission.status.undetermined" = "Not requested yet";
"onboarding.permission.status.granted" = "Allowed";
"onboarding.permission.status.denied" = "Denied";
"legal.privacyPolicy" = "Privacy Policy";
"onboarding.enable.title" = "Enable OSGKeyboard";
"onboarding.enable.step1" = "Settings → General → Keyboard → Keyboards";
"onboarding.enable.step2" = "Tap “Add New Keyboard…” and choose OSGKeyboard";
@@ -95,6 +109,10 @@
"settings.onDeviceOnly.body" = "Disable cloud fallback. The keyboard reports an error instead of going online.";
"settings.legend.onDevice" = "On-device";
"settings.legend.cloudFallback" = "Cloud fallback";
"settings.privacy.title" = "Privacy";
"settings.privacy.policy" = "Privacy Policy";
"settings.privacy.fullAccess.title" = "About Full Access";
"settings.privacy.fullAccess.body" = "Full Access is required for the microphone and to read your API key. OSGKeyboard does not record or upload what you type with the keyboard.";
/* App group error */
"appGroup.error.title" = "App Group not configured";
@@ -105,8 +123,9 @@
/* Keyboard preview */
"preview.title" = "Keyboard Preview";
"preview.subtitle" = "Tap the disc to start/stop recording. The real keyboard uses the same layout.";
"preview.placeholder" = "Type or tap to record";
"preview.subtitle" = "Tap Start/Stop Recording to test here. On the real keyboard, tap the mic disc to dictate.";
"preview.placeholder" = "Type here or tap the record button";
"preview.placeholder" = "Type here or tap the record button";
"preview.clear" = "Clear text";
"preview.openSettingsA11y" = "Open OSGKeyboard settings";
"preview.modeChip.cycle" = "Cycle input mode";
@@ -118,7 +137,7 @@
"preview.error.engineStart" = "Engine start failed: %@";
/* Keyboard (ext) */
"keyboard.placeholder.idle" = "Hold to talk";
"keyboard.placeholder.idle" = "Tap to talk";
"keyboard.placeholder.preparing" = "Preparing";
"keyboard.placeholder.processing" = "Processing";
"keyboard.placeholder.error" = "Polishing failed";
@@ -129,7 +148,7 @@
"keyboard.denied.speech" = "Speech denied";
"keyboard.openSettingsA11y" = "Open OSGKeyboard settings";
"keyboard.deniedHint" = "Opens the OSGKeyboard settings page where you can grant microphone or speech recognition access.";
"keyboard.pressToTalkA11y" = "Push to talk";
"keyboard.tapToTalkA11y" = "Tap to talk";
/* Mode chip labels (used in both ext + preview stub) */
"mode.off" = "Off";
@@ -146,3 +165,21 @@
"locale.en-US" = "English (US)";
"locale.ja-JP" = "Japanese";
"locale.ko-KR" = "Korean";
/* Flow session */
"flow.error.noSpeech" = "No speech detected. Please try again.";
"keyboard.denied.mic" = "Microphone access denied";
"keyboard.denied.speech" = "Speech recognition denied";
/* Home · Flow session */
"home.flow.active" = "Voice session active";
"home.flow.inactive" = "Voice session inactive";
"home.flow.hint" = "Voice session starts automatically. Switch to any app and tap the keyboard mic to dictate.";
"home.flow.starting" = "Starting voice session…";
"home.flow.openSettings" = "Open Settings to grant permissions";
"home.flow.start" = "Start voice session";
"home.flow.end" = "End voice session";
"flow.error.speechRequired" = "Speech recognition access is required for voice sessions.";
"flow.error.micRequired" = "Microphone access is required for background voice sessions.";
"flow.error.micUnavailable" = "Microphone is unavailable on this device.";
"flow.error.audioUnavailable" = "Could not start background audio.";
+39 -3
View File
@@ -3,7 +3,21 @@
"OSGKeyboard" = "OSGKeyboard";
/* Onboarding */
"onboarding.welcome.subtitle" = "按住说话,松开即得润色文字。";
"onboarding.welcome.subtitle" = "点按说话,再点结束 — 任意 App 里都能获得润色文字。";
"onboarding.enable.fullAccessNote" = "完全访问用于麦克风与 API 配置读取。我们不会记录或上传你的击键内容。";
"onboarding.permission.mic.title" = "麦克风权限";
"onboarding.permission.mic.body" = "语音输入需要麦克风。音频在设备端转录,不会上传原始录音。";
"onboarding.permission.mic.allow" = "允许麦克风";
"onboarding.permission.mic.deniedHint" = "麦克风被拒绝。可前往设置开启,或点「下一步」继续。";
"onboarding.permission.speech.title" = "语音识别权限";
"onboarding.permission.speech.body" = "端侧语音识别将语音转为文字。音频不会离开你的设备。";
"onboarding.permission.speech.allow" = "允许语音识别";
"onboarding.permission.speech.deniedHint" = "语音识别被拒绝。可前往设置开启,或点「下一步」继续。";
"onboarding.permission.openSettings" = "打开设置";
"onboarding.permission.status.undetermined" = "尚未请求";
"onboarding.permission.status.granted" = "已允许";
"onboarding.permission.status.denied" = "已拒绝";
"legal.privacyPolicy" = "隐私政策";
"onboarding.enable.title" = "启用 OSGKeyboard";
"onboarding.enable.step1" = "设置 → 通用 → 键盘 → 键盘";
"onboarding.enable.step2" = "点击「添加新键盘…」并选择 OSGKeyboard";
@@ -95,6 +109,10 @@
"settings.onDeviceOnly.body" = "禁用云端回退,识别失败时会报错而非联网。";
"settings.legend.onDevice" = "端侧识别";
"settings.legend.cloudFallback" = "需联网";
"settings.privacy.title" = "隐私";
"settings.privacy.policy" = "隐私政策";
"settings.privacy.fullAccess.title" = "关于完全访问";
"settings.privacy.fullAccess.body" = "完全访问用于麦克风与读取 API Key。OSGKeyboard 不会记录或上传你在键盘上的击键内容。";
/* App group error */
"appGroup.error.title" = "App Group 未配置";
@@ -105,8 +123,8 @@
/* Keyboard preview */
"preview.title" = "键盘预览";
"preview.subtitle" = "点按 disc 开始/结束录音;真实键盘使用同样布局。";
"preview.placeholder" = "试着输入或按 disc 录音";
"preview.subtitle" = "点击「开始录音」/「停止录音」按钮测试;真实键盘为长按麦克风圆盘。";
"preview.placeholder" = "试着输入或点击按钮录音";
"preview.clear" = "清空";
"preview.openSettingsA11y" = "打开 OSGKeyboard 设置";
"preview.modeChip.cycle" = "切换输入模式";
@@ -146,3 +164,21 @@
"locale.en-US" = "English (US)";
"locale.ja-JP" = "日本語";
"locale.ko-KR" = "한국어";
/* Flow session */
"flow.error.noSpeech" = "未检测到语音,请重试。";
"keyboard.denied.mic" = "麦克风权限被拒绝";
"keyboard.denied.speech" = "语音识别权限被拒绝";
/* Home · Flow session */
"home.flow.active" = "语音会话进行中";
"home.flow.inactive" = "语音会话未启动";
"home.flow.hint" = "语音会话会自动启动。切到任意 App,点按键盘麦克风即可说话。";
"home.flow.starting" = "正在启动语音会话…";
"home.flow.openSettings" = "前往设置授予权限";
"home.flow.start" = "启动语音会话";
"home.flow.end" = "结束语音会话";
"flow.error.speechRequired" = "需要语音识别权限才能使用语音会话。";
"flow.error.micRequired" = "需要麦克风权限才能维持后台语音会话。";
"flow.error.micUnavailable" = "当前设备无法使用麦克风。";
"flow.error.audioUnavailable" = "无法启动后台音频。";
+2 -2
View File
@@ -39,8 +39,8 @@
<string>$(PRODUCT_MODULE_NAME).KeyboardViewController</string>
</dict>
<key>NSMicrophoneUsageDescription</key>
<string>OSGKeyboard needs microphone access to transcribe your voice into text.</string>
<string>OSGKeyboard uses the microphone for voice dictation and keeps a background audio session active while a voice session is running.</string>
<key>NSSpeechRecognitionUsageDescription</key>
<string>OSGKeyboard uses on-device speech recognition to transcribe your dictation. Audio never leaves your device.</string>
<string>OSGKeyboard uses on-device speech recognition to transcribe your voice. Audio is processed on your device and is not uploaded for transcription.</string>
</dict>
</plist>
+343 -121
View File
@@ -4,7 +4,7 @@
// Principal class for the Custom Keyboard Extension. Hosts a single
// SwiftUI tree (`KeyboardRootView`) and drives the recording pipeline:
//
// AudioCaptureService ASRService PolishingService insertText
// host app dictation handoff App Group transcript insertText
//
// Design notes:
// The class is `@MainActor` every UI mutation and `textDocumentProxy`
@@ -19,12 +19,22 @@
import UIKit
import SwiftUI
import AVFoundation
import OSGKeyboardShared
@objc(KeyboardViewController)
@MainActor
public final class KeyboardViewController: UIInputViewController {
private enum FlowWatchdog {
static let pollIntervalNs: UInt64 = 200_000_000
/// Give the user time to manually open the host app when auto-jump fails.
static let startTimeout: TimeInterval = 30
static let resultTimeout: TimeInterval = 45
}
private enum DictationWatchdog {
static let pollIntervalNs: UInt64 = 400_000_000
static let timeout: TimeInterval = 45
}
// MARK: - View model
@@ -37,17 +47,21 @@ public final class KeyboardViewController: UIInputViewController {
// MARK: - State
private let state = State()
private let audio = AudioCaptureService()
private let asr: ASRService = ASRServiceFactory.make()
private let polisher = PolishingService()
private let permissions = PermissionManager()
private let persistor = AppGroupPersistor()
private var session: AudioCaptureService.Session?
private var asrTask: Task<Void, Never>?
private var levelTask: Task<Void, Never>?
private var hosting: UIHostingController<KeyboardRootView>!
/// Legacy one-shot handoff (`osgkeyboard://dictate`).
private var awaitingDictationResult = false
private var dictationRequestStartedAt: TimeInterval = 0
private var dictationWatchdogTask: Task<Void, Never>?
/// Flow session: waiting for host app to come alive after `startflow`.
private var isPendingFlowStart = false
private var flowStartDeadline: TimeInterval = 0
private var isFlowRecording = false
private var flowWatchdogTask: Task<Void, Never>?
private var utteranceTimerTask: Task<Void, Never>?
private var utteranceStartedAt: TimeInterval = 0
// MARK: - Lifecycle
@@ -60,6 +74,8 @@ public final class KeyboardViewController: UIInputViewController {
installStateActions()
installSwiftUI()
loadPersistedConfig()
consumePendingDictationResultIfNeeded()
refreshDictationProgressStateIfNeeded()
}
public override func viewWillDisappear(_ animated: Bool) {
@@ -67,6 +83,12 @@ public final class KeyboardViewController: UIInputViewController {
cancelPipeline()
}
public override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
consumePendingDictationResultIfNeeded()
refreshDictationProgressStateIfNeeded()
}
public override func didReceiveMemoryWarning() {
super.didReceiveMemoryWarning()
cancelPipeline()
@@ -74,7 +96,8 @@ public final class KeyboardViewController: UIInputViewController {
public override func textDidChange(_ textInput: (any UITextInput)?) {
super.textDidChange(textInput)
// Hook for future per-app mode switching (e.g. password field .off).
consumePendingDictationResultIfNeeded()
refreshDictationProgressStateIfNeeded()
}
// MARK: - Wiring
@@ -82,7 +105,7 @@ public final class KeyboardViewController: UIInputViewController {
private func installStateActions() {
state.beginRecording = { [weak self] in self?.pressBegan() }
state.endRecording = { [weak self] in self?.pressEnded() }
state.tapMic = { [weak self] in self?.advanceToNextInputMode() }
state.tapMic = { [weak self] in self?.toggleRecording() }
state.openSettings = { [weak self] in self?.openHostApp() }
state.setMode = { [weak self] m in self?.persistMode(m) }
state.setLocale = { [weak self] l in self?.persistLocale(l) }
@@ -125,11 +148,18 @@ public final class KeyboardViewController: UIInputViewController {
// MARK: - Press handlers
private func toggleRecording() {
switch state.phase {
case .recording:
pressEnded()
case .idle, .denied, .error:
pressBegan()
case .requestingPermissions, .processing:
break
}
}
private func pressBegan() {
// Allow re-entry from `.denied` and from a finished/cleared
// `.error` so the user can simply press the mic again after
// returning from Settings with permission granted they
// shouldn't have to wait for an auto-clear timer.
switch state.phase {
case .idle, .denied, .error:
break
@@ -137,106 +167,213 @@ public final class KeyboardViewController: UIInputViewController {
return
}
guard state.mode != .off else { return }
// Set the intermediate phase SYNCHRONOUSLY so a rapid second
// press (before the first Task has had a chance to flip phase to
// .recording) is rejected by the guard above. This fixes the race
// where the user double-tapped the mic and we started two
// pipelines at once.
state.phase = .requestingPermissions
Task { @MainActor [weak self] in
guard let self else { return }
let micGranted = await self.permissions.requestMicPermission()
guard micGranted else {
self.state.phase = .denied(.mic)
return
}
// We explicitly ask for Speech recognition permission here.
// `SpeechAnalyzer` does not expose a dedicated request API,
// so the app still relies on the shared Speech permission
// gate and `NSSpeechRecognitionUsageDescription`.
let speechGranted = await self.permissions.requestSpeechPermission()
guard speechGranted else {
self.state.phase = .denied(.speech)
return
}
self.startPipeline()
guard hasFullAccess else {
let msg = "请在系统设置中为 OSGKeyboard 开启“允许完全访问”,否则无法使用语音输入"
state.phase = .error(.unknown(msg), message: msg)
scheduleAutoClearError()
return
}
guard AppGroup.isAvailable else {
let msg = "App Group 未配置,键盘无法与主 App 通信。请重新安装并检查签名配置。"
state.phase = .error(.appGroupUnavailable, message: msg)
scheduleAutoClearError()
return
}
if FlowSessionBridge.isSessionActive() {
startFlowRecording()
} else {
beginFlowStart()
}
}
private func pressEnded() {
guard state.phase == .recording else { return }
stopPipeline()
if isPendingFlowStart {
cancelPendingFlowStart()
return
}
guard isFlowRecording else { return }
isFlowRecording = false
stopUtteranceCountdown()
FlowSessionBridge.setRecordingState(.stopped)
state.phase = .processing
state.lastTranscript = "识别中..."
startFlowResultWatchdog()
}
// MARK: - Pipeline
private func startFlowRecording() {
isPendingFlowStart = false
flowStartDeadline = 0
stopFlowWatchdog()
private func startPipeline() {
let session = audio.start()
self.session = session
state.phase = .recording
state.level = 0
FlowSessionBridge.setTranscriptionLanguage(state.localeId)
FlowSessionBridge.setRecordingState(.recording)
isFlowRecording = true
state.lastTranscript = ""
state.phase = .recording
startUtteranceCountdown()
startFlowLevelWatchdog()
debug("startFlowRecording")
}
let locale = resolveLocale(state.localeId)
let events = asr.transcribe(
stream: session.audio,
locale: locale
)
asrTask = Task { @MainActor [weak self] in
guard let self else { return }
var lastPartial: String = ""
for await event in events {
switch event {
case .capability(let onDevice):
self.state.onDeviceSupported = onDevice
case .partial(let s):
lastPartial = s
self.state.lastTranscript = s
case .final(let s):
let transcript = s.isEmpty ? lastPartial : s
self.handleFinalTranscript(transcript)
case .error(let m):
self.state.phase = .error(.asr(m))
self.scheduleAutoClearError()
private func startUtteranceCountdown() {
utteranceStartedAt = Date().timeIntervalSince1970
state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration)
utteranceTimerTask?.cancel()
utteranceTimerTask = Task { @MainActor [weak self] in
while let self, self.isFlowRecording, !Task.isCancelled {
let elapsed = Date().timeIntervalSince1970 - self.utteranceStartedAt
let remaining = max(0, Int(ceil(FlowSessionKeys.maxUtteranceDuration - elapsed)))
self.state.utteranceRemainingSeconds = remaining
if remaining <= 0 {
self.pressEnded()
return
}
}
}
levelTask = Task { @MainActor [weak self] in
for await level in session.levels {
guard let self else { return }
// Smooth a little extra to feel natural.
self.state.level = Double(self.state.level) * 0.6 + Double(level.meter) * 0.4
try? await Task.sleep(nanoseconds: 200_000_000)
}
}
}
private func stopPipeline() {
session?.stop()
session = nil
asrTask?.cancel(); asrTask = nil
levelTask?.cancel(); levelTask = nil
private func stopUtteranceCountdown() {
utteranceTimerTask?.cancel()
utteranceTimerTask = nil
state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration)
}
private func beginFlowStart() {
isPendingFlowStart = true
isFlowRecording = false
flowStartDeadline = Date().timeIntervalSince1970 + FlowWatchdog.startTimeout
state.lastTranscript = "正在启动语音会话..."
state.phase = .processing
openHostApp(path: "startflow")
startFlowStartWatchdog()
debug("beginFlowStart")
}
private func cancelPendingFlowStart() {
isPendingFlowStart = false
flowStartDeadline = 0
stopFlowWatchdog()
state.phase = .idle
state.lastTranscript = ""
}
private func startFlowStartWatchdog() {
stopFlowWatchdog()
flowWatchdogTask = Task { @MainActor [weak self] in
while let self, !Task.isCancelled, self.isPendingFlowStart {
if FlowSessionBridge.isSessionActive() {
self.startFlowRecording()
return
}
let now = Date().timeIntervalSince1970
if self.flowStartDeadline > 0, now > self.flowStartDeadline {
self.isPendingFlowStart = false
self.flowStartDeadline = 0
self.showManualSettingsHint(path: "startflow")
return
}
try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs)
}
}
}
private func startFlowLevelWatchdog() {
stopFlowWatchdog()
flowWatchdogTask = Task { @MainActor [weak self] in
while let self, !Task.isCancelled, self.isFlowRecording {
let levels = FlowSessionBridge.audioLevels()
if let peak = levels.max(), peak > 0 {
self.state.level = Double(peak)
}
try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs)
}
}
}
private func startFlowResultWatchdog() {
stopFlowWatchdog()
let startedAt = Date().timeIntervalSince1970
flowWatchdogTask = Task { @MainActor [weak self] in
while let self, !Task.isCancelled {
if let result = FlowSessionBridge.consumeTranscriptionResult() {
self.stopFlowWatchdog()
self.handleFlowTranscript(result)
return
}
if let error = FlowSessionBridge.consumeTranscriptionError() {
self.stopFlowWatchdog()
self.state.phase = .error(.unknown(error), message: error)
self.scheduleAutoClearError()
return
}
let now = Date().timeIntervalSince1970
if now - startedAt > FlowWatchdog.resultTimeout {
self.stopFlowWatchdog()
let msg = "等待识别结果超时,请重试"
self.state.phase = .error(.unknown(msg), message: msg)
self.scheduleAutoClearError()
return
}
try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs)
}
}
}
private func handleFlowTranscript(_ transcript: String) {
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else {
state.phase = .idle
state.level = 0
return
}
// Host app already polished when configured; keyboard only inserts.
textDocumentProxy.insertText(trimmed)
state.lastTranscript = ""
state.level = 0
state.phase = .idle
debug("flow insert length=\(trimmed.count)")
}
private func stopFlowWatchdog() {
flowWatchdogTask?.cancel()
flowWatchdogTask = nil
}
private func cancelPipeline() {
stopPipeline()
asr.cancel()
if state.phase == .recording || state.phase == .processing {
if isFlowRecording || isPendingFlowStart {
if isFlowRecording {
FlowSessionBridge.setRecordingState(.aborted)
}
isFlowRecording = false
isPendingFlowStart = false
stopUtteranceCountdown()
stopFlowWatchdog()
state.level = 0
}
if awaitingDictationResult {
debug("cancelPipeline ignored while awaiting legacy handoff result")
return
}
if state.phase == .processing {
state.phase = .idle
}
state.level = 0
// Reset the on-device flag so the StatusBadge stops showing the
// cloud-fallback indicator between recordings.
state.onDeviceSupported = false
}
private func handleFinalTranscript(_ transcript: String) {
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else {
debug("received empty transcript")
awaitingDictationResult = false
stopDictationWatchdog()
state.phase = .idle
return
}
debug("received transcript length=\(trimmed.count)")
awaitingDictationResult = false
stopDictationWatchdog()
// Local engine or transcribe mode: insert directly, no LLM call.
if state.isLocalEngine || state.mode == .transcribe {
textDocumentProxy.insertText(trimmed)
@@ -307,18 +444,8 @@ public final class KeyboardViewController: UIInputViewController {
persistor.persist(mode: m)
if isRecording {
if m == .off {
// Switching to .off while recording: drop the partial
// (no insertion, no LLM). User has explicitly disabled
// the keyboard, so we honour that immediately.
stopPipeline()
state.phase = .idle
state.lastTranscript = ""
} else if m == .transcribe {
// Switching to .transcribe while in .polish: end the
// recording, the partial will flow through
// handleFinalTranscript which inserts the raw text in
// .transcribe mode (no LLM call).
pressEnded()
}
}
}
@@ -335,34 +462,123 @@ public final class KeyboardViewController: UIInputViewController {
// MARK: - Open host app
private func openHostApp() {
let urlString = "osgkeyboard://settings"
if let url = URL(string: urlString) {
var responder: UIResponder? = self
while let r = responder {
if let app = r as? UIApplication {
app.open(url)
return
}
responder = r.next
}
private func openHostApp(path: String = "settings") {
guard hasFullAccess else {
let msg = "未开启“允许完全访问”,请先在键盘设置中打开"
state.phase = .error(.unknown(msg), message: msg)
scheduleAutoClearError()
return
}
if let url = URL(string: UIApplication.openSettingsURLString) {
var responder: UIResponder? = self
while let r = responder {
if let app = r as? UIApplication {
app.open(url); return
}
responder = r.next
guard let url = URL(string: "osgkeyboard://\(path)") else {
handleHostAppOpenResult(path: path, success: false)
return
}
HostAppLauncher.open(url: url, from: self) { [weak self] success in
self?.handleHostAppOpenResult(path: path, success: success)
}
}
private func handleHostAppOpenResult(path: String, success: Bool) {
debug("openHostApp path=\(path) success=\(success)")
guard !success else { return }
// Flow start: auto-jump often fails in WeChat/Safari keep polling
// so a manually opened host app can still satisfy the session check.
if path == "startflow", isPendingFlowStart {
state.lastTranscript = "无法自动跳转,请从主屏幕打开 OSGKeyboard,然后返回继续"
return
}
if path == "dictate" {
awaitingDictationResult = false
stopDictationWatchdog()
}
showManualSettingsHint(path: path)
}
private func consumePendingDictationResultIfNeeded() {
guard let transcript = DictationBridge.consumePendingTranscript() else { return }
debug("consumePendingDictationResultIfNeeded success")
handleFinalTranscript(transcript)
}
private func refreshDictationProgressStateIfNeeded() {
guard awaitingDictationResult, case .processing = state.phase else { return }
let progress = DictationBridge.currentStatus()
switch progress.status {
case .requested:
state.lastTranscript = state.isLocalEngine
? "正在打开 OSGKeyboard(本地转写)..."
: "正在打开 OSGKeyboard..."
case .recording:
state.lastTranscript = state.isLocalEngine
? "正在本地录音,请完成后返回当前输入页"
: "正在录音,请完成后返回当前输入页"
case .transcribing:
state.lastTranscript = state.isLocalEngine
? "本地识别中,请稍候并返回输入页"
: "识别中,请稍候并返回输入页"
case .error:
let msg = progress.message ?? "录音失败,请重试"
debug("host returned error: \(msg)")
awaitingDictationResult = false
stopDictationWatchdog()
state.phase = .error(.unknown(msg), message: msg)
scheduleAutoClearError()
case .cancelled:
debug("host cancelled")
awaitingDictationResult = false
stopDictationWatchdog()
state.phase = .idle
case .done, .idle:
break
}
// Host app can be killed or leave without callback. If status does not
// advance for too long, fail fast with an actionable retry message.
let now = Date().timeIntervalSince1970
let lastProgressAt = progress.updatedAt > 0 ? progress.updatedAt : dictationRequestStartedAt
if now - lastProgressAt > DictationWatchdog.timeout {
let timeoutMessage = "等待录音结果超时,请返回 OSGKeyboard 完成录音后重试"
debug("dictation timeout after \(Int(now - lastProgressAt))s")
awaitingDictationResult = false
stopDictationWatchdog()
DictationBridge.clear()
state.phase = .error(.unknown(timeoutMessage), message: timeoutMessage)
scheduleAutoClearError()
}
}
private func showManualSettingsHint(path: String = "settings") {
let msg: String
if !hasFullAccess {
msg = "请先开启 OSGKeyboard 的“允许完全访问”,否则键盘无法跳转到 App"
} else if path == "settings" {
msg = "系统拒绝了键盘跳转。请手动打开 OSGKeyboard App 进入设置页"
} else if path == "startflow" {
msg = "语音会话未启动。请从主屏幕打开 OSGKeyboard App,返回后再按麦克风"
} else if state.isLocalEngine {
msg = "系统拒绝了键盘跳转。请先手动打开 OSGKeyboard 完成本地转写,再返回输入页"
} else {
msg = "系统拒绝了键盘跳转。请先手动打开 OSGKeyboard 录音,再返回输入页"
}
state.phase = .error(.unknown(msg), message: msg)
scheduleAutoClearError()
}
private func startDictationWatchdog() {
stopDictationWatchdog()
dictationWatchdogTask = Task { @MainActor [weak self] in
while let self, !Task.isCancelled, self.awaitingDictationResult {
self.consumePendingDictationResultIfNeeded()
self.refreshDictationProgressStateIfNeeded()
try? await Task.sleep(nanoseconds: DictationWatchdog.pollIntervalNs)
}
}
}
// MARK: - Helpers
private func resolveLocale(_ id: String) -> Locale {
if id == "auto" { return .current }
return Locale(identifier: id)
private func stopDictationWatchdog() {
dictationWatchdogTask?.cancel()
dictationWatchdogTask = nil
}
private func scheduleAutoClearError() {
@@ -381,4 +597,10 @@ public final class KeyboardViewController: UIInputViewController {
}
}
}
private func debug(_ message: String) {
#if DEBUG
print("🎙️[KeyboardVC] \(message)")
#endif
}
}
+2 -2
View File
@@ -6,9 +6,9 @@
<array>
<string>group.com.osgkeyboard.shared</string>
</array>
<key>com.apple.security.keychain-access-groups</key>
<key>keychain-access-groups</key>
<array>
<string>com.osgkeyboard.shared</string>
<string>$(AppIdentifierPrefix)com.osgkeyboard.shared</string>
</array>
</dict>
</plist>
@@ -62,16 +62,19 @@ public struct AppGroupPersistor {
/// Persist `mode` to the App Group store.
public func persist(mode: KeyboardViewController.State.InputMode) {
guard AppGroup.isAvailable else { return }
AppGroupStore().setModeId(mode.rawValue)
}
/// Persist `localeId` to the App Group store.
public func persist(localeId: String) {
guard AppGroup.isAvailable else { return }
AppGroupStore().setLocaleId(localeId)
}
/// Persist `engineMode` to the App Group store.
public func persist(engineMode: String) {
guard AppGroup.isAvailable else { return }
AppGroupStore().setEngineMode(engineMode)
}
}
@@ -0,0 +1,70 @@
// HostAppLauncher.swift
// OSGKeyboard · Keyboard Extension
//
// Opens the host app via URL using every extension-safe strategy:
// 1. `extensionContext.open` (official)
// 2. Responder-chain `UIApplication.open` (TypeWhisper pattern)
// 3. `sharedApplication` KVC fallback (common in full-access keyboards)
import UIKit
enum HostAppLauncher {
@MainActor
static func open(
url: URL,
from controller: KeyboardViewController,
completion: @escaping @MainActor (Bool) -> Void
) {
if let context = controller.extensionContext {
context.open(url) { success in
Task { @MainActor in
if success {
completion(true)
return
}
completion(openViaFallback(url, from: controller))
}
}
return
}
completion(openViaFallback(url, from: controller))
}
@MainActor
private static func openViaFallback(
_ url: URL,
from controller: KeyboardViewController
) -> Bool {
if openViaResponderChain(url, from: controller) {
return true
}
return openViaSharedApplication(url)
}
@MainActor
private static func openViaResponderChain(
_ url: URL,
from controller: KeyboardViewController
) -> Bool {
var responder: UIResponder? = controller
while let current = responder {
if let application = current as? UIApplication {
application.open(url, options: [:]) { _ in }
return true
}
responder = current.next
}
return false
}
@MainActor
private static func openViaSharedApplication(_ url: URL) -> Bool {
guard
let application = UIApplication.value(forKeyPath: "sharedApplication") as? UIApplication
else {
return false
}
application.open(url, options: [:]) { _ in }
return true
}
}
@@ -47,15 +47,34 @@ public final class PermissionManager: @unchecked Sendable {
/// method of its own and the framework checks the same TCC
/// entry on first use.
public func requestSpeechPermission() async -> Bool {
await Self.requestSpeechPermissionNonisolated()
}
// MARK: - Nonisolated permission bridge
//
// `SFSpeechRecognizer.requestAuthorization` callback is not guaranteed
// to run on main queue. Building the callback inline inside a
// `@MainActor` method can trigger runtime actor/isolation assertions.
// Keep the continuation + callback creation in nonisolated helpers.
private nonisolated static func requestSpeechPermissionNonisolated() async -> Bool {
await withCheckedContinuation { (cont: CheckedContinuation<Bool, Never>) in
SFSpeechRecognizer.requestAuthorization { status in
switch status {
case .authorized: cont.resume(returning: true)
case .denied, .restricted, .notDetermined:
cont.resume(returning: false)
@unknown default:
cont.resume(returning: false)
}
SFSpeechRecognizer.requestAuthorization(
makeSpeechAuthHandler(continuation: cont)
)
}
}
private nonisolated static func makeSpeechAuthHandler(
continuation: CheckedContinuation<Bool, Never>
) -> @Sendable (SFSpeechRecognizerAuthorizationStatus) -> Void {
return { status in
switch status {
case .authorized:
continuation.resume(returning: true)
case .denied, .restricted, .notDetermined:
continuation.resume(returning: false)
@unknown default:
continuation.resume(returning: false)
}
}
}
+51 -32
View File
@@ -22,7 +22,7 @@ import SwiftUI
import OSGKeyboardShared
public struct KeyboardRootView: View {
@Environment(\.themePalette) private var palette: ThemePalette
@Environment(\.colorScheme) private var colorScheme
@ObservedObject var state: State
@@ -35,6 +35,10 @@ public struct KeyboardRootView: View {
/// it up.
static let totalHeight: CGFloat = 280
private var palette: ThemePalette {
colorScheme == .dark ? Palette.dark : Palette.light
}
public var body: some View {
VStack(spacing: 0) {
topBar
@@ -48,26 +52,12 @@ public struct KeyboardRootView: View {
}
.padding(.top, 4)
.padding(.bottom, 6)
// iOS keyboard extensions always render dark (Apple's default
// for custom keyboards), and we let the system UI chrome show
// through by drawing no background of our own.
// Let the system UI chrome show through by drawing no background
// of our own.
.background(Color.clear)
.frame(height: Self.totalHeight)
// Top edge: subtle highlight gradient + 0.5pt divider line.
// These give the keyboard a "physical surface" feel and visually
// separate it from the host text field above. We overlay (not
// background) so the underlying color stays clear.
.overlay(alignment: .top) {
VStack(spacing: 0) {
Rectangle()
.fill(LinearGradient(
colors: [Color.white.opacity(0.05), .clear],
startPoint: .top, endPoint: .bottom
))
.frame(height: 1)
Spacer(minLength: 0)
}
}
// Feed the resolved palette to all nested chips/buttons.
.environment(\.themePalette, palette)
}
// MARK: - Top bar
@@ -96,7 +86,7 @@ public struct KeyboardRootView: View {
.overlay(Circle().stroke(palette.divider, lineWidth: 0.5))
}
.buttonStyle(.plain)
.accessibilityLabel(Text("home.action.openSettingsA11y"))
.accessibilityLabel(Text(KeyboardL10n.openSettingsA11y))
}
.padding(.horizontal, Spacing.md)
}
@@ -115,9 +105,8 @@ public struct KeyboardRootView: View {
RecordButton(
phase: buttonPhase,
level: state.level,
onPressBegan: state.beginRecording,
onPressEnded: state.endRecording,
onTap: state.tapMic
remainingSeconds: state.phase == .recording ? state.utteranceRemainingSeconds : nil,
onToggle: state.tapMic
)
.frame(width: 140, height: 140)
}
@@ -139,7 +128,7 @@ public struct KeyboardRootView: View {
state.deleteBackward()
}
Button(action: state.insertSpace) {
Text("common.space")
Text(KeyboardL10n.space)
.font(TypeStyle.body)
.foregroundStyle(palette.textPrimary)
.frame(maxWidth: .infinity, minHeight: 42)
@@ -150,7 +139,7 @@ public struct KeyboardRootView: View {
)
}
.buttonStyle(.plain)
.accessibilityLabel(Text("common.space"))
.accessibilityLabel(Text(KeyboardL10n.space))
ToolbarIconButton(systemName: "return", label: "newline") {
state.insertNewline()
}
@@ -211,13 +200,13 @@ private struct TranscriptLine: View {
ZStack {
switch phase {
case .idle:
Text("keyboard.placeholder.idle")
Text(KeyboardL10n.placeholderIdle)
.font(TypeStyle.caption)
.foregroundStyle(palette.textTertiary)
case .requestingPermissions:
HStack(spacing: 6) {
ProgressView().controlSize(.mini).tint(palette.textSecondary)
Text("keyboard.placeholder.preparing")
Text(KeyboardL10n.placeholderPreparing)
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
}
@@ -231,9 +220,11 @@ private struct TranscriptLine: View {
case .processing:
HStack(spacing: 6) {
ProgressView().controlSize(.mini).tint(palette.accent)
Text("keyboard.placeholder.processing")
Text(transcript.isEmpty ? KeyboardL10n.placeholderProcessing : transcript)
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
.lineLimit(1)
.truncationMode(.tail)
}
case .error(_, let msg):
Text(msg ?? "")
@@ -256,7 +247,7 @@ private struct TranscriptLine: View {
.contentShape(Rectangle())
}
.buttonStyle(.plain)
.accessibilityHint(Text("keyboard.deniedHint"))
.accessibilityHint(Text(KeyboardL10n.deniedHint))
}
}
.frame(maxWidth: .infinity)
@@ -265,8 +256,8 @@ private struct TranscriptLine: View {
private func deniedMessage(for reason: KeyboardViewController.State.Phase.Reason) -> String {
switch reason {
case .mic: return "麦克风被拒绝 · Mic denied"
case .speech: return "语音识别被拒绝 · Speech denied"
case .mic: return KeyboardL10n.micDenied
case .speech: return KeyboardL10n.speechDenied
}
}
}
@@ -361,7 +352,7 @@ private struct LocalEngineChip: View {
var body: some View {
HStack(spacing: 4) {
Image(systemName: "iphone.badge.checkmark")
Text("keyboard.placeholder.localBadge")
Text(KeyboardL10n.localBadge)
}
.font(TypeStyle.caption2)
.foregroundStyle(palette.accent)
@@ -372,6 +363,34 @@ private struct LocalEngineChip: View {
}
}
// MARK: - Extension text fallback
//
// Custom keyboard extensions can end up without the expected localized
// resource table when signing/project generation drifts. Keep a tiny
// in-code fallback map so UI never shows raw key names like
// "common.space" in production.
private enum KeyboardL10n {
private static var isChinese: Bool {
Locale.preferredLanguages.first?.hasPrefix("zh") == true
}
static var space: String { isChinese ? "空格" : "Space" }
static var placeholderIdle: String { isChinese ? "点按说话" : "Tap to talk" }
static var placeholderPreparing: String { isChinese ? "准备中…" : "Preparing" }
static var placeholderProcessing: String { isChinese ? "处理中…" : "Processing" }
static var localBadge: String { isChinese ? "本地" : "On-device" }
static var micDenied: String { isChinese ? "麦克风被拒绝" : "Mic denied" }
static var speechDenied: String { isChinese ? "语音识别被拒绝" : "Speech denied" }
static var deniedHint: String {
isChinese
? "打开 OSGKeyboard 设置,可授予麦克风 / 语音识别权限。"
: "Open OSGKeyboard settings to grant microphone / speech access."
}
static var openSettingsA11y: String {
isChinese ? "打开 OSGKeyboard 设置" : "Open OSGKeyboard settings"
}
}
// MARK: - Mode chip
private struct ModeChip: View {
+42 -78
View File
@@ -1,10 +1,8 @@
// RecordButton.swift
// OSGKeyboard · Keyboard Extension
//
// The hero control. 120 pt primary disc with a soft inner gradient, a
// breathing outer ring while recording, and a centred waveform that maps
// directly to the real audio RMS. Idle / recording / processing are three
// distinct visual states no flicker, no surprise transitions.
// Tap-to-toggle mic: tap once to start, tap again to stop. Shows a
// remaining-time countdown while recording; last 10 seconds turn red.
import SwiftUI
import OSGKeyboardShared
@@ -21,38 +19,38 @@ struct RecordButton: View {
let phase: Phase
let level: Double // 0...1
let onPressBegan: () -> Void
let onPressEnded: () -> Void
let onTap: () -> Void
/// Seconds left in the current utterance; shown only while recording.
let remainingSeconds: Int?
let onToggle: () -> Void
@GestureState private var isPressed: Bool = false
@State private var breath: Bool = false
init(
phase: Phase,
level: Double,
onPressBegan: @escaping () -> Void,
onPressEnded: @escaping () -> Void,
onTap: @escaping () -> Void
remainingSeconds: Int? = nil,
onToggle: @escaping () -> Void
) {
self.phase = phase
self.level = level
self.onPressBegan = onPressBegan
self.onPressEnded = onPressEnded
self.onTap = onTap
self.remainingSeconds = remainingSeconds
self.onToggle = onToggle
}
private var isUrgent: Bool {
guard phase == .recording, let remainingSeconds else { return false }
return remainingSeconds <= 10
}
var body: some View {
ZStack {
// Outer breathing ring (recording only)
Circle()
.stroke(palette.recordRed.opacity(0.35), lineWidth: 2)
.stroke(palette.recordRed.opacity(isUrgent ? 0.55 : 0.35), lineWidth: isUrgent ? 3 : 2)
.frame(width: 150, height: 150)
.scaleEffect(breath ? 1.18 : 0.95)
.opacity(phase == .recording ? 1 : 0)
.animation(Motion.breath, value: breath)
// Halo: soft red glow that intensifies with input level
Circle()
.fill(
RadialGradient(
@@ -68,7 +66,6 @@ struct RecordButton: View {
.animation(Motion.soft, value: phase)
.animation(Motion.soft, value: level)
// Secondary outer ring (always present, dimmer when idle)
Circle()
.stroke(
Color.white.opacity(phase == .idle ? 0.08 : 0.12),
@@ -76,7 +73,6 @@ struct RecordButton: View {
)
.frame(width: 140, height: 140)
// Main disc with gradient + soft inner highlight
ZStack {
Circle()
.fill(discGradient)
@@ -84,7 +80,6 @@ struct RecordButton: View {
.stroke(Color.white.opacity(0.16), lineWidth: 1)
.blendMode(.overlay)
// Centre content switches by phase
Group {
switch phase {
case .idle:
@@ -92,17 +87,19 @@ struct RecordButton: View {
.font(.system(size: 38, weight: .medium))
.foregroundStyle(palette.textPrimary)
case .recording:
WaveformView(level: level, active: true)
.frame(width: 80, height: 44)
.transition(.opacity)
VStack(spacing: 4) {
if let remainingSeconds {
Text(formatRemaining(remainingSeconds))
.font(.system(size: 22, weight: .semibold, design: .rounded))
.foregroundStyle(isUrgent ? .white : palette.textPrimary)
.monospacedDigit()
.contentTransition(.numericText())
}
WaveformView(level: level, active: true)
.frame(width: 72, height: 32)
}
.transition(.opacity)
case .processing:
// Scaled to ~50pt inside a 120pt disc (~42%)
// same absolute size as the preview stub's
// spinner (2.5x of the default ProgressView),
// just a smaller fraction because the real
// disc is bigger. The user gets the same
// visual weight whether they're looking at
// the in-app preview or the live keyboard.
ProgressView()
.progressViewStyle(.circular)
.tint(palette.textPrimary)
@@ -115,63 +112,34 @@ struct RecordButton: View {
}
}
.frame(width: 120, height: 120)
.scaleEffect(isPressed ? 0.94 : 1.0)
.animation(Motion.quick, value: isPressed)
.animation(Motion.soft, value: phase)
.animation(Motion.soft, value: remainingSeconds)
}
.contentShape(Circle())
// Press-to-talk: act on the FIRST touch-down, not after a 150 ms
// minimum duration. That's what Typeless feels like, and it's what
// makes the keyboard feel responsive. A tap (very short press) is
// interpreted as "toggle" for the secondary action (onTap), not
// "record" the recording only fires if the press lasts long
// enough to read as intentional. This avoids the previous bug
// where every single tap fired both onPressBegan AND onTap.
.gesture(
LongPressGesture(minimumDuration: 0.18)
.sequenced(before: DragGesture(minimumDistance: 0))
.updating($isPressed) { value, state, _ in
switch value {
case .second(true, _): state = true
default: state = false
}
}
.onChanged { value in
if case .second(true, _) = value, !pressArmed {
pressArmed = true
onPressBegan()
}
}
.onEnded { _ in
if pressArmed { pressArmed = false; onPressEnded() }
}
)
.simultaneousGesture(
// Pure tap: only fires when the user lifts before the long-press
// threshold. This becomes the "secondary action" (e.g. cycle
// mode). It is paired with, not conflicting with, the long-press.
TapGesture(count: 1)
.onEnded {
if !pressArmed { onTap() }
}
)
.onTapGesture {
guard phase != .processing else { return }
onToggle()
}
.onAppear { breath = (phase == .recording) }
.onChange(of: phase) { _, new in
breath = (new == .recording)
}
.accessibilityLabel(Text("keyboard.pressToTalkA11y"))
.accessibilityLabel(Text("keyboard.tapToTalkA11y"))
}
@State private var pressArmed: Bool = false
private func formatRemaining(_ seconds: Int) -> String {
let m = seconds / 60
let s = seconds % 60
return String(format: "%d:%02d", m, s)
}
private var discGradient: LinearGradient {
switch phase {
case .recording:
return LinearGradient(
colors: [palette.recordRed.opacity(0.95), palette.recordRed.opacity(0.75)],
startPoint: .top,
endPoint: .bottom
)
let colors: [Color] = isUrgent
? [palette.recordRed, palette.recordRed.opacity(0.85)]
: [palette.recordRed.opacity(0.95), palette.recordRed.opacity(0.75)]
return LinearGradient(colors: colors, startPoint: .top, endPoint: .bottom)
case .processing:
return LinearGradient(
colors: [palette.surfaceElevated, palette.surface],
@@ -185,10 +153,6 @@ struct RecordButton: View {
endPoint: .bottom
)
case .idle:
// Brand green same hue as `Palette.{dark,light}.accent`
// and the AccentColor asset. The disc is the keyboard's
// primary CTA, and a dark-gray disc looked like an inert
// surface, not an actionable button.
return LinearGradient(
colors: [
palette.accent.opacity(0.95),
+4 -4
View File
@@ -105,15 +105,15 @@
/* Keyboard preview */
"preview.title" = "Keyboard Preview";
"preview.subtitle" = "Tap the disc to start/stop recording. The real keyboard uses the same layout.";
"preview.placeholder" = "Type or tap to record";
"preview.subtitle" = "Tap Start/Stop Recording to test here. On the real keyboard, press and hold the mic disc.";
"preview.placeholder" = "Type here or tap the record button";
"preview.clear" = "Clear text";
"preview.openSettingsA11y" = "Open OSGKeyboard settings";
"preview.modeChip.cycle" = "Cycle input mode";
"preview.localeChip.cycle" = "Cycle recognition language";
/* Keyboard (ext) */
"keyboard.placeholder.idle" = "Hold to talk";
"keyboard.placeholder.idle" = "Tap to talk";
"keyboard.placeholder.preparing" = "Preparing";
"keyboard.placeholder.processing" = "Processing";
"keyboard.placeholder.error" = "Polishing failed";
@@ -124,7 +124,7 @@
"keyboard.denied.speech" = "Speech denied";
"keyboard.openSettingsA11y" = "Open OSGKeyboard settings";
"keyboard.deniedHint" = "Opens the OSGKeyboard settings page where you can grant microphone or speech recognition access.";
"keyboard.pressToTalkA11y" = "Push to talk";
"keyboard.tapToTalkA11y" = "Tap to talk";
/* Mode chip labels (used in both ext + preview stub) */
"mode.off" = "Off";
@@ -105,15 +105,15 @@
/* Keyboard preview */
"preview.title" = "键盘预览";
"preview.subtitle" = "点按 disc 开始/结束录音;真实键盘使用同样布局。";
"preview.placeholder" = "试着输入或按 disc 录音";
"preview.subtitle" = "点击「开始录音」/「停止录音」按钮测试;真实键盘为长按麦克风圆盘。";
"preview.placeholder" = "试着输入或点击按钮录音";
"preview.clear" = "清空";
"preview.openSettingsA11y" = "打开 OSGKeyboard 设置";
"preview.modeChip.cycle" = "切换输入模式";
"preview.localeChip.cycle" = "切换识别语言";
/* Keyboard (ext) */
"keyboard.placeholder.idle" = "按说话";
"keyboard.placeholder.idle" = "按说话";
"keyboard.placeholder.preparing" = "准备中…";
"keyboard.placeholder.processing" = "处理中…";
"keyboard.placeholder.error" = "润色失败";
@@ -124,7 +124,7 @@
"keyboard.denied.speech" = "语音识别被拒绝";
"keyboard.openSettingsA11y" = "打开 OSGKeyboard 设置";
"keyboard.deniedHint" = "打开 OSGKeyboard 设置,可授予麦克风 / 语音识别权限。";
"keyboard.pressToTalkA11y" = "按说话";
"keyboard.tapToTalkA11y" = "按说话";
/* Mode chip labels */
"mode.off" = "关闭";
+8 -11
View File
@@ -12,18 +12,15 @@ public enum AppGroup {
/// Whether the App Group container is available on this device.
///
/// Cached at first read the underlying `UserDefaults(suiteName:)`
/// call is cheap, but main-app startup and every keyboard-extension
/// read hit it, so we memoize the result.
///
/// Production code paths MUST go through `isAvailable` first and
/// surface a friendly error view (e.g. `AppGroupErrorView`) on the
/// main app, or the keyboard extension's persisted-locale load.
/// Calling `defaults` directly when the group is missing will trip
/// the DEBUG `fatalError` below that path is reserved for
/// developer-only escape hatches and intentional debugging.
/// Checks both `UserDefaults(suiteName:)` *and* the on-disk container.
/// The suite alone can appear to open while the container is still `(null)`
/// when provisioning is misconfigured that case produces the
/// `CFPrefsPlistSource Container: (null)` console warning.
public static let isAvailable: Bool = {
UserDefaults(suiteName: identifier) != nil
guard UserDefaults(suiteName: identifier) != nil else { return false }
return FileManager.default.containerURL(
forSecurityApplicationGroupIdentifier: identifier
) != nil
}()
/// Shared UserDefaults instance for cross-process config.
@@ -0,0 +1,26 @@
// EngineServiceLabel.swift
// OSGKeyboard · Shared
//
// Human-readable summary of the active engine / AI provider for UI hints.
import Foundation
public enum EngineServiceLabel {
public static func summary(
engineMode: String,
providerId: String,
model: String
) -> String {
let isChinese = Locale.preferredLanguages.first?.hasPrefix("zh") == true
let prefix = isChinese ? "当前:" : "Active: "
if engineMode == "local" {
return isChinese
? "\(prefix)本地引擎 · Apple SpeechAnalyzer"
: "\(prefix)On-device · Apple SpeechAnalyzer"
}
let provider = LLMProvider.provider(id: providerId)
let trimmedModel = model.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmedModel.isEmpty { return "\(prefix)\(provider.name)" }
return "\(prefix)\(provider.name) · \(trimmedModel)"
}
}
+20 -11
View File
@@ -27,6 +27,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
static let modeId = "config.modeId"
static let localeId = "config.localeId"
static let engineMode = "config.engineMode"
static let hasCompletedOnboarding = "config.hasCompletedOnboarding"
}
@Published public var providerId: String {
@@ -66,6 +67,9 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
@Published public var engineMode: String {
didSet { defaults.set(engineMode, forKey: Key.engineMode) }
}
@Published public var hasCompletedOnboarding: Bool {
didSet { defaults.set(hasCompletedOnboarding, forKey: Key.hasCompletedOnboarding) }
}
public var isConfigured: Bool {
// Local engine (on-device ASR only) doesn't need an API key,
@@ -73,10 +77,13 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
// Treat it as always-configured so onboarding's "Next" button
// enables the moment the user picks the local path, instead
// of forcing them to fill in cloud fields they won't use.
if engineMode == "local" { return true }
if isLocalEngine { return true }
return !baseURL.isEmpty && !apiKey.isEmpty && !model.isEmpty
}
/// On-device ASR only no cloud LLM polish.
public var isLocalEngine: Bool { engineMode == "local" }
/// The system prompt the user *sees* in the editor fall back to the
/// provider-aware default from `AppGroupStore` when nothing is set.
public var defaultSystemPrompt: String {
@@ -85,25 +92,27 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
private let defaults: UserDefaults
public init(defaults: UserDefaults = AppGroup.defaults) {
self.defaults = defaults
let pid = defaults.string(forKey: Key.providerId) ?? "openai"
public init(defaults: UserDefaults? = nil) {
let resolvedDefaults: UserDefaults = defaults ?? (AppGroup.isAvailable ? AppGroup.defaults : .standard)
self.defaults = resolvedDefaults
let pid = resolvedDefaults.string(forKey: Key.providerId) ?? "openai"
let preset = LLMProvider.provider(id: pid)
self.providerId = pid
self.baseURL = defaults.string(forKey: Key.baseURL) ?? preset.defaultBaseURL
self.baseURL = resolvedDefaults.string(forKey: Key.baseURL) ?? preset.defaultBaseURL
// Resolve the API key with a one-shot migration from the legacy
// UserDefaults slot. After this runs once, `Key.apiKeyLegacy`
// is empty in the suite and all subsequent reads go through the
// Keychain.
self.apiKey = ProviderConfig.resolveAPIKey(defaults: defaults)
self.apiKey = ProviderConfig.resolveAPIKey(defaults: resolvedDefaults)
self.model = defaults.string(forKey: Key.model) ?? preset.defaultModel
self.systemPrompt = defaults.string(forKey: Key.systemPrompt)
self.model = resolvedDefaults.string(forKey: Key.model) ?? preset.defaultModel
self.systemPrompt = resolvedDefaults.string(forKey: Key.systemPrompt)
?? AppGroupStore.defaultSystemPrompt(for: pid)
self.modeId = defaults.string(forKey: Key.modeId) ?? "polish"
self.localeId = defaults.string(forKey: Key.localeId) ?? "auto"
self.engineMode = defaults.string(forKey: Key.engineMode) ?? "cloud"
self.modeId = resolvedDefaults.string(forKey: Key.modeId) ?? "polish"
self.localeId = resolvedDefaults.string(forKey: Key.localeId) ?? "auto"
self.engineMode = resolvedDefaults.string(forKey: Key.engineMode) ?? "cloud"
self.hasCompletedOnboarding = resolvedDefaults.bool(forKey: Key.hasCompletedOnboarding)
}
/// Read the API key from the Keychain, falling back to a one-time
+147 -87
View File
@@ -114,72 +114,111 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
private let lock = OSAllocatedUnfairLock()
private var analyzer: SpeechAnalyzer?
private var analyzerTask: Task<Void, Never>?
private var analyzerFinished = false
/// Canonical capture format: 16 kHz mono Float32 from `AudioCaptureService`
/// / `PreviewASRController` before it reaches SpeechAnalyzer.
private static let captureFormat = AVAudioFormat(
commonFormat: .pcmFormatFloat32,
sampleRate: 16_000,
channels: 1,
interleaved: false
)!
func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent> {
AsyncStream { continuation in
// SpeechAnalyzer is always fully on-device.
continuation.yield(.capability(onDeviceSupported: true))
let transcriber = DictationTranscriber(locale: locale, preset: .progressiveShortDictation)
let newAnalyzer = SpeechAnalyzer(modules: [transcriber])
self.lock.withLock { self.analyzer = newAnalyzer }
// iOS 26's `DictationTranscriber` requires **Int16** PCM
// (precondition `"Audio sample data must be 16-bit signed
// integers"` the legacy recognizer used Float32 at this
// boundary; `SpeechAnalyzer` is strict Int16). 16 kHz mono, Int16,
// interleaved the canonical layout Apple's Speech
// framework examples use.
let audioFormat = AVAudioFormat(
commonFormat: .pcmFormatInt16,
sampleRate: 16_000,
channels: 1,
interleaved: true
)!
let task = Task { [weak self] in
guard let self else { return }
do {
try await newAnalyzer.prepareToAnalyze(in: audioFormat)
let inputStream = self.makeInputStream(from: stream, format: audioFormat)
// Feed audio in a child task so we can concurrently
// iterate `transcriber.results` on the outer task.
// After the audio stream ends, finalize so the results
// sequence can drain and complete.
let feedTask = Task {
do {
try await newAnalyzer.start(inputSequence: inputStream)
try await newAnalyzer.finalizeAndFinishThroughEndOfInput()
} catch {}
self.lock.withLock { self.analyzerFinished = false }
defer {
self.lock.withLock {
self.analyzer = nil
self.analyzerTask = nil
self.analyzerFinished = true
}
defer { feedTask.cancel() }
}
var lastText = ""
do {
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
Self.debug("locale unsupported: \(locale.identifier(.bcp47))")
continuation.yield(.error("当前系统未分配可用语音语言模型,请稍后重试或切换语言"))
continuation.finish()
return
}
let transcriber = DictationTranscriber(locale: resolvedLocale, preset: .progressiveShortDictation)
do {
try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
} catch {
Self.debug("asset prepare failed: \(error.localizedDescription)")
continuation.yield(.error("语音语言资源未就绪,请稍后重试"))
continuation.finish()
return
}
let newAnalyzer = SpeechAnalyzer(modules: [transcriber])
self.lock.withLock { self.analyzer = newAnalyzer }
guard let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(
compatibleWith: [transcriber],
considering: Self.captureFormat
) else {
continuation.yield(.error("当前设备不支持该语音输入格式"))
continuation.finish()
return
}
try await newAnalyzer.prepareToAnalyze(in: analyzerFormat)
let inputStream = self.makeInputStream(from: stream, analyzerFormat: analyzerFormat)
// Apple recommends consuming `transcriber.results` concurrently
// while `analyzeSequence` drains the input stream.
let resultsTask = Task<String, Error> {
var lastText = ""
for try await result in transcriber.results {
if Task.isCancelled { break }
// `result.text` is an AttributedString; extract plain text.
let text = result.text.characters.map(String.init).joined()
let text = String(result.text.characters)
guard !text.isEmpty, text != lastText else { continue }
lastText = text
continuation.yield(.partial(text))
}
} catch {
// Results sequence threw likely cancellation.
return lastText
}
if !Task.isCancelled {
continuation.yield(.final(lastText))
let lastSampleTime = try await newAnalyzer.analyzeSequence(inputStream)
if let lastSampleTime {
try await newAnalyzer.finalizeAndFinish(through: lastSampleTime)
} else {
try await newAnalyzer.cancelAndFinishNow()
}
let lastText: String
do {
lastText = try await resultsTask.value
} catch {
Self.debug("transcriber results failed: \(error.localizedDescription)")
continuation.yield(.error(error.localizedDescription))
continuation.finish()
return
}
let trimmed = lastText.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty {
continuation.yield(.error("未识别到语音内容,请重试"))
} else {
continuation.yield(.final(trimmed))
}
continuation.finish()
} catch is CancellationError {
continuation.finish()
} catch {
Self.debug("SpeechAnalyzer failed: \(error.localizedDescription)")
continuation.yield(.error(error.localizedDescription))
continuation.finish()
}
@@ -192,64 +231,52 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
}
}
func cancel() {
let (task, currentAnalyzer) = lock.withLock { () -> (Task<Void, Never>?, SpeechAnalyzer?) in
let t = analyzerTask
let a = analyzer
analyzerTask = nil
analyzer = nil
return (t, a)
private static func debug(_ message: String) {
#if DEBUG
print("🎙️[ASRService] \(message)")
#endif
}
private static func prepareAssetsIfNeeded(
for transcriber: DictationTranscriber,
locale: Locale
) async throws {
do {
_ = try await AssetInventory.reserve(locale: locale)
} catch {
// Reservation may already exist or slots are full; continue.
}
task?.cancel()
if let a = currentAnalyzer {
Task { await a.cancelAndFinishNow() }
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
try await request.downloadAndInstall()
}
}
/// Maps the `AudioBufferSnapshot` stream into the `AnalyzerInput` stream
/// that `SpeechAnalyzer` consumes.
///
/// `AudioBufferSnapshot.samples` is `[Float]` (the transport format
/// both `AudioCaptureService` and `PreviewASRController` produce
/// Float32 is what `AVAudioEngine` gives us at the hardware rate
/// and we already downsample to 16 kHz mono before this point).
/// iOS 26's `DictationTranscriber` requires **Int16** PCM at the
/// `AnalyzerInput` boundary, so we convert per-snapshot here.
///
/// The conversion is the textbook `[-1.0, 1.0]` × 32767 + clip +
/// cast. For a 16 kHz mono feed the loop is ~16k iters/sec
/// well under any audio-thread budget so a simple scalar loop
/// beats pulling in `vDSP` (which would also need a scratch
/// buffer the audio thread can't easily allocate).
func cancel() {
let (task, currentAnalyzer, finished) = lock.withLock { () -> (Task<Void, Never>?, SpeechAnalyzer?, Bool) in
let t = analyzerTask
let a = analyzer
let f = analyzerFinished
analyzerTask = nil
analyzer = nil
return (t, a, f)
}
task?.cancel()
guard !finished, let currentAnalyzer else { return }
Task { await currentAnalyzer.cancelAndFinishNow() }
}
/// Maps 16 kHz Float32 snapshots into `AnalyzerInput` using the format
/// returned by `bestAvailableAudioFormat(compatibleWith:considering:)`.
private func makeInputStream(
from stream: AsyncStream<AudioBufferSnapshot>,
format: AVAudioFormat
analyzerFormat: AVAudioFormat
) -> AsyncStream<AnalyzerInput> {
AsyncStream { continuation in
Task {
for await snap in stream {
guard !snap.samples.isEmpty else { continue }
let capacity = AVAudioFrameCount(snap.samples.count)
guard let pcm = AVAudioPCMBuffer(
pcmFormat: format,
frameCapacity: capacity
) else { continue }
pcm.frameLength = capacity
// For a 1-channel Int16 buffer (interleaved or not
// single channel, so the data layout is identical),
// `int16ChannelData?[0]` gives us the raw sample
// pointer. Clip on overflow to avoid wraparound
// (a Float like 1.5 would otherwise become a
// negative Int16 after the implicit truncation).
if let dst = pcm.int16ChannelData?[0] {
snap.samples.withUnsafeBufferPointer { src in
ASRServiceFactory.convertFloat32ToInt16(
source: src.baseAddress,
sourceCount: src.count,
destination: dst
)
}
guard let pcm = Self.makeAnalyzerPCMBuffer(from: snap, format: analyzerFormat) else {
continue
}
continuation.yield(AnalyzerInput(buffer: pcm))
}
@@ -257,4 +284,37 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
}
}
}
private static func makeAnalyzerPCMBuffer(
from snap: AudioBufferSnapshot,
format: AVAudioFormat
) -> AVAudioPCMBuffer? {
let capacity = AVAudioFrameCount(snap.samples.count)
guard capacity > 0,
let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else {
return nil
}
pcm.frameLength = capacity
switch format.commonFormat {
case .pcmFormatInt16:
guard let dst = pcm.int16ChannelData?[0] else { return nil }
snap.samples.withUnsafeBufferPointer { src in
ASRServiceFactory.convertFloat32ToInt16(
source: src.baseAddress,
sourceCount: src.count,
destination: dst
)
}
case .pcmFormatFloat32:
guard let dst = pcm.floatChannelData?[0] else { return nil }
snap.samples.withUnsafeBufferPointer { src in
guard let base = src.baseAddress else { return }
memcpy(dst, base, src.count * MemoryLayout<Float>.stride)
}
default:
return nil
}
return pcm
}
}
@@ -14,8 +14,15 @@ import Foundation
public struct AppGroupStore: @unchecked Sendable {
public let defaults: UserDefaults
public init(defaults: UserDefaults = AppGroup.defaults) {
self.defaults = defaults
public init(defaults: UserDefaults? = nil) {
if let defaults {
self.defaults = defaults
return
}
// Never hard-crash on implicit construction sites (e.g. default
// service initializers). If App Group is unavailable, use .standard
// so callers can still surface a user-facing setup error.
self.defaults = AppGroup.isAvailable ? AppGroup.defaults : .standard
}
// MARK: - Keys
@@ -0,0 +1,106 @@
// DictationBridge.swift
// OSGKeyboard · Shared
//
// Lightweight App Group bridge for host-app dictation handoff:
// keyboard extension -> open host app for recording
// host app -> writes final transcript
// keyboard extension -> consumes pending transcript and inserts text
import Foundation
public enum DictationBridge {
public enum Status: String, Sendable, Equatable {
case idle
case requested
case recording
case transcribing
case done
case cancelled
case error
}
private enum Key {
static let pendingText = "dictation.pendingText"
static let updatedAt = "dictation.updatedAt"
static let status = "dictation.status"
static let statusUpdatedAt = "dictation.statusUpdatedAt"
static let statusMessage = "dictation.statusMessage"
}
private static func resolvedDefaults(_ defaults: UserDefaults?) -> UserDefaults {
if let defaults {
return defaults
}
return AppGroup.isAvailable ? AppGroup.defaults : .standard
}
public static func setStatus(
_ status: Status,
message: String? = nil,
defaults: UserDefaults? = nil
) {
let store = resolvedDefaults(defaults)
store.set(status.rawValue, forKey: Key.status)
store.set(Date().timeIntervalSince1970, forKey: Key.statusUpdatedAt)
if let message, !message.isEmpty {
store.set(message, forKey: Key.statusMessage)
} else {
store.removeObject(forKey: Key.statusMessage)
}
}
public static func currentStatus(
defaults: UserDefaults? = nil
) -> (status: Status, message: String?, updatedAt: TimeInterval) {
let store = resolvedDefaults(defaults)
let raw = store.string(forKey: Key.status) ?? Status.idle.rawValue
let status = Status(rawValue: raw) ?? .idle
let message = store.string(forKey: Key.statusMessage)
let updatedAt = store.double(forKey: Key.statusUpdatedAt)
return (status, message, updatedAt)
}
public static func markRequested(defaults: UserDefaults? = nil) {
let store = resolvedDefaults(defaults)
store.removeObject(forKey: Key.pendingText)
setStatus(.requested, defaults: store)
}
/// Store a transcript for the keyboard extension to consume.
public static func storePendingTranscript(_ text: String, defaults: UserDefaults? = nil) {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
let store = resolvedDefaults(defaults)
store.set(trimmed, forKey: Key.pendingText)
store.set(Date().timeIntervalSince1970, forKey: Key.updatedAt)
setStatus(.done, defaults: store)
}
/// Returns and clears the pending transcript if present.
public static func consumePendingTranscript(
maxAge: TimeInterval = 180,
defaults: UserDefaults? = nil
) -> String? {
let store = resolvedDefaults(defaults)
guard let text = store.string(forKey: Key.pendingText) else {
return nil
}
if maxAge > 0 {
let ts = store.double(forKey: Key.updatedAt)
if ts > 0, Date().timeIntervalSince1970 - ts > maxAge {
clear(defaults: store)
return nil
}
}
store.removeObject(forKey: Key.pendingText)
setStatus(.idle, defaults: store)
return text
}
public static func clear(defaults: UserDefaults? = nil) {
let store = resolvedDefaults(defaults)
store.removeObject(forKey: Key.pendingText)
store.removeObject(forKey: Key.updatedAt)
setStatus(.idle, defaults: store)
}
}
@@ -0,0 +1,270 @@
// FlowContinuousCapture.swift
// OSGKeyboard · Shared
//
// TypeWhisper-style continuous mic capture for Flow sessions: one
// AVAudioEngine + input tap for the entire session. Utterances gate
// whether buffers are forwarded to ASR; levels are always computed on
// the audio thread and read from the main thread (never UserDefaults
// from the realtime tap that caused cross-process crashes).
import Foundation
import AVFoundation
import os
private enum FlowCaptureConstants {
static let levelBarCount = 24
static let targetSampleRate: Double = 16_000
}
/// Thread-safe relay for utterance-scoped ASR snapshots.
private final class FlowCaptureStreamRelay: @unchecked Sendable {
private let lock = OSAllocatedUnfairLock()
private var continuation: AsyncStream<AudioBufferSnapshot>.Continuation?
func bind(_ continuation: AsyncStream<AudioBufferSnapshot>.Continuation) {
lock.withLock { self.continuation = continuation }
}
func yield(_ snapshot: AudioBufferSnapshot) {
lock.withLock { continuation?.yield(snapshot) }
}
func finish() {
lock.withLock {
continuation?.finish()
continuation = nil
}
}
}
/// Rolling bar levels updated from the audio tap; read on the main actor.
private final class FlowLevelStore: @unchecked Sendable {
private let lock = OSAllocatedUnfairLock()
private var levels: [Float]
init(barCount: Int) {
levels = Array(repeating: 0, count: barCount)
}
func update(from buffer: AVAudioPCMBuffer, barCount: Int) {
let computed = Self.calculateLevels(from: buffer, barCount: barCount)
lock.withLock { levels = computed }
}
func snapshot() -> [Float] {
lock.withLock { levels }
}
private static func calculateLevels(from buffer: AVAudioPCMBuffer, barCount: Int) -> [Float] {
guard let channelData = buffer.floatChannelData else {
return Array(repeating: 0, count: barCount)
}
let frameLength = Int(buffer.frameLength)
guard frameLength > 0 else {
return Array(repeating: 0, count: barCount)
}
let samplesPerBar = max(frameLength / barCount, 1)
var result = [Float]()
result.reserveCapacity(barCount)
for barIndex in 0..<barCount {
let start = barIndex * samplesPerBar
let end = min(start + samplesPerBar, frameLength)
var sum: Float = 0
for i in start..<end {
sum += abs(channelData[0][i])
}
let avg = sum / Float(max(end - start, 1))
result.append(min(avg * 50, 1))
}
return result
}
}
@MainActor
public final class FlowContinuousCapture {
public enum StartError: LocalizedError {
case invalidHardwareFormat(sampleRate: Double, channels: Int)
case formatCreateFailed
case converterCreateFailed
case engineStartFailed(String)
case audioSessionFailed(String)
public var errorDescription: String? {
switch self {
case .invalidHardwareFormat(let sr, let ch):
return String.localizedStringWithFormat(
NSLocalizedString("preview.error.micUnavailable", comment: ""),
sr,
ch
)
case .formatCreateFailed:
return NSLocalizedString("preview.error.formatCreate", comment: "")
case .converterCreateFailed:
return NSLocalizedString("preview.error.converterCreate", comment: "")
case .engineStartFailed(let detail):
return String.localizedStringWithFormat(
NSLocalizedString("preview.error.engineStart", comment: ""),
detail
)
case .audioSessionFailed(let detail):
return String.localizedStringWithFormat(
NSLocalizedString("preview.error.audioSession", comment: ""),
detail
)
}
}
}
public static let levelBarCount = FlowCaptureConstants.levelBarCount
private let audioEngine = AVAudioEngine()
private let streamRelay = FlowCaptureStreamRelay()
private let levelStore = FlowLevelStore(barCount: FlowCaptureConstants.levelBarCount)
private let isUtteranceActive = OSAllocatedUnfairLock(initialState: false)
private var didInstallTap = false
private var isRunning = false
public init() {}
public var running: Bool { isRunning }
/// Configure `.playAndRecord`, install a permanent input tap, start the engine.
public func start() throws {
guard !isRunning else { return }
let session = AVAudioSession.sharedInstance()
do {
try session.setCategory(
.playAndRecord,
mode: .measurement,
options: [.defaultToSpeaker, .allowBluetoothHFP, .mixWithOthers]
)
try session.setActive(true, options: .notifyOthersOnDeactivation)
} catch {
throw StartError.audioSessionFailed(error.localizedDescription)
}
let inputNode = audioEngine.inputNode
let hwFormat = inputNode.outputFormat(forBus: 0)
guard hwFormat.sampleRate > 0, hwFormat.channelCount > 0 else {
throw StartError.invalidHardwareFormat(
sampleRate: hwFormat.sampleRate,
channels: Int(hwFormat.channelCount)
)
}
guard let targetFormat = AVAudioFormat(
commonFormat: .pcmFormatFloat32,
sampleRate: FlowCaptureConstants.targetSampleRate,
channels: 1,
interleaved: false
) else {
throw StartError.formatCreateFailed
}
guard let converter = AVAudioConverter(from: hwFormat, to: targetFormat) else {
throw StartError.converterCreateFailed
}
if !didInstallTap {
let utteranceFlag = isUtteranceActive
let relay = streamRelay
let levels = levelStore
let tap = Self.makeAudioTapBlock(
converter: converter,
targetFormat: targetFormat,
hwFormat: hwFormat,
utteranceFlag: utteranceFlag,
levelStore: levels,
streamRelay: relay
)
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
didInstallTap = true
}
audioEngine.prepare()
do {
try audioEngine.start()
} catch {
throw StartError.engineStartFailed(error.localizedDescription)
}
isRunning = true
}
/// Tear down the engine and release the audio session.
public func stop() {
isUtteranceActive.withLock { $0 = false }
streamRelay.finish()
if didInstallTap {
audioEngine.inputNode.removeTap(onBus: 0)
didInstallTap = false
}
if audioEngine.isRunning {
audioEngine.stop()
}
isRunning = false
try? AVAudioSession.sharedInstance().setActive(
false,
options: .notifyOthersOnDeactivation
)
}
/// Begin forwarding downsampled buffers to ASR for one utterance.
public func beginUtterance() -> AsyncStream<AudioBufferSnapshot> {
isUtteranceActive.withLock { $0 = true }
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
streamRelay.bind(continuation)
return stream
}
/// Stop forwarding buffers; finishes the ASR stream.
public func endUtterance() {
isUtteranceActive.withLock { $0 = false }
streamRelay.finish()
}
public func cancelUtterance() {
endUtterance()
}
public func currentAudioLevels() -> [Float] {
levelStore.snapshot()
}
// MARK: - Audio tap (nonisolated runs on realtime thread)
private nonisolated static func makeAudioTapBlock(
converter: AVAudioConverter,
targetFormat: AVAudioFormat,
hwFormat: AVAudioFormat,
utteranceFlag: OSAllocatedUnfairLock<Bool>,
levelStore: FlowLevelStore,
streamRelay: FlowCaptureStreamRelay
) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
return { buffer, _ in
levelStore.update(from: buffer, barCount: FlowCaptureConstants.levelBarCount)
guard utteranceFlag.withLock({ $0 }) else { return }
let outFrames = AVAudioFrameCount(
Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate
)
guard outFrames > 0,
let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outFrames)
else { return }
var error: NSError?
let status = converter.convert(to: outBuffer, error: &error) { _, outStatus in
outStatus.pointee = .haveData
return buffer
}
guard status == .haveData, error == nil, outBuffer.frameLength > 0 else { return }
let snapshot = AudioBufferSnapshot(buffer: outBuffer)
guard !snapshot.samples.isEmpty else { return }
streamRelay.yield(snapshot)
}
}
}
@@ -0,0 +1,217 @@
// FlowSessionBridge.swift
// OSGKeyboard · Shared
//
// TypeWhisper-style Flow session bridge: keyboard writes recording
// signals; host app writes transcription results. Legacy one-shot
// dictation handoff remains in `DictationBridge`.
import Foundation
public enum FlowSessionBridge {
private static func resolvedDefaults(_ defaults: UserDefaults?) -> UserDefaults {
if let defaults { return defaults }
return AppGroup.isAvailable ? AppGroup.defaults : .standard
}
/// Force cross-process visibility. Must only be called on the main thread.
private static func flush(_ store: UserDefaults) {
if Thread.isMainThread {
store.synchronize()
}
}
// MARK: - Session lifecycle (host app)
public static func markSessionActive(
duration: TimeInterval = FlowSessionKeys.defaultSessionDuration,
defaults: UserDefaults? = nil
) {
let store = resolvedDefaults(defaults)
let expires = Date().timeIntervalSince1970 + duration
store.set(true, forKey: FlowSessionKeys.flowSessionActive)
store.set(expires, forKey: FlowSessionKeys.flowSessionExpires)
writeHeartbeat(defaults: store)
setRecordingState(.idle, defaults: store)
clearTranscription(defaults: store)
flush(store)
}
public static func markSessionInactive(defaults: UserDefaults? = nil) {
let store = resolvedDefaults(defaults)
store.set(false, forKey: FlowSessionKeys.flowSessionActive)
store.removeObject(forKey: FlowSessionKeys.flowSessionExpires)
store.removeObject(forKey: FlowSessionKeys.flowHeartbeat)
setRecordingState(.idle, defaults: store)
clearTranscription(defaults: store)
flush(store)
}
public static func writeHeartbeat(defaults: UserDefaults? = nil) {
let store = resolvedDefaults(defaults)
store.set(Date().timeIntervalSince1970, forKey: FlowSessionKeys.flowHeartbeat)
flush(store)
}
public static func extendSession(
by duration: TimeInterval = FlowSessionKeys.defaultSessionDuration,
defaults: UserDefaults? = nil
) {
let store = resolvedDefaults(defaults)
let expires = Date().timeIntervalSince1970 + duration
store.set(true, forKey: FlowSessionKeys.flowSessionActive)
store.set(expires, forKey: FlowSessionKeys.flowSessionExpires)
flush(store)
}
// MARK: - Session validity (keyboard)
/// True when expires is in the future and heartbeat is fresh.
public static func isSessionActive(defaults: UserDefaults? = nil) -> Bool {
let store = resolvedDefaults(defaults)
flush(store)
let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires)
guard expires > Date().timeIntervalSince1970 else { return false }
let heartbeat = store.double(forKey: FlowSessionKeys.flowHeartbeat)
guard heartbeat > 0 else { return false }
let staleness = Date().timeIntervalSince1970 - heartbeat
return staleness <= FlowSessionKeys.heartbeatStaleInterval
}
public static func sessionExpiresAt(defaults: UserDefaults? = nil) -> TimeInterval? {
let store = resolvedDefaults(defaults)
let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires)
return expires > 0 ? expires : nil
}
/// Seconds until session expiry; nil when expired or never started.
public static func remainingSessionDuration(defaults: UserDefaults? = nil) -> TimeInterval? {
guard let expires = sessionExpiresAt(defaults: defaults) else { return nil }
let remaining = expires - Date().timeIntervalSince1970
return remaining > 0 ? remaining : nil
}
// MARK: - Recording signals (keyboard host)
public static func setRecordingState(
_ state: FlowSessionKeys.RecordingState,
defaults: UserDefaults? = nil
) {
let store = resolvedDefaults(defaults)
store.set(state.rawValue, forKey: FlowSessionKeys.keyboardRecordingState)
flush(store)
}
public static func recordingState(
defaults: UserDefaults? = nil
) -> FlowSessionKeys.RecordingState {
let store = resolvedDefaults(defaults)
let raw = store.string(forKey: FlowSessionKeys.keyboardRecordingState) ?? FlowSessionKeys.RecordingState.idle.rawValue
return FlowSessionKeys.RecordingState(rawValue: raw) ?? .idle
}
public static func setTranscriptionLanguage(
_ localeId: String,
defaults: UserDefaults? = nil
) {
let store = resolvedDefaults(defaults)
store.set(localeId, forKey: FlowSessionKeys.transcriptionLanguage)
flush(store)
}
// MARK: - Results (host keyboard)
public static func storeTranscriptionResult(
_ text: String,
defaults: UserDefaults? = nil
) {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
let store = resolvedDefaults(defaults)
store.set(trimmed, forKey: FlowSessionKeys.transcriptionResult)
store.removeObject(forKey: FlowSessionKeys.transcriptionError)
setRecordingState(.idle, defaults: store)
flush(store)
}
public static func storeTranscriptionError(
_ message: String,
defaults: UserDefaults? = nil
) {
let store = resolvedDefaults(defaults)
store.set(message, forKey: FlowSessionKeys.transcriptionError)
setRecordingState(.idle, defaults: store)
flush(store)
}
/// Returns and clears a pending transcription result, if any.
public static func consumeTranscriptionResult(defaults: UserDefaults? = nil) -> String? {
let store = resolvedDefaults(defaults)
flush(store)
guard let text = store.string(forKey: FlowSessionKeys.transcriptionResult), !text.isEmpty else {
return nil
}
store.removeObject(forKey: FlowSessionKeys.transcriptionResult)
flush(store)
return text
}
/// Returns and clears a pending transcription error, if any.
public static func consumeTranscriptionError(defaults: UserDefaults? = nil) -> String? {
let store = resolvedDefaults(defaults)
flush(store)
guard let message = store.string(forKey: FlowSessionKeys.transcriptionError), !message.isEmpty else {
return nil
}
store.removeObject(forKey: FlowSessionKeys.transcriptionError)
flush(store)
return message
}
public static func audioLevels(defaults: UserDefaults? = nil) -> [Float] {
let store = resolvedDefaults(defaults)
flush(store)
if let levels = store.array(forKey: FlowSessionKeys.audioLevels) as? [Double], !levels.isEmpty {
return levels.map { Float($0) }
}
if let levels = store.array(forKey: FlowSessionKeys.audioLevels) as? [NSNumber], !levels.isEmpty {
return levels.map { $0.floatValue }
}
return []
}
/// Host app: publish waveform bars for the keyboard (main thread only).
public static func storeAudioLevels(
_ levels: [Float],
defaults: UserDefaults? = nil
) {
let store = resolvedDefaults(defaults)
store.set(levels.map { Double($0) }, forKey: FlowSessionKeys.audioLevels)
flush(store)
}
/// Clear pending result/error before a new utterance.
public static func clearPendingTranscription(defaults: UserDefaults? = nil) {
let store = resolvedDefaults(defaults)
clearTranscription(defaults: store)
flush(store)
}
public static func clearFlowState(defaults: UserDefaults? = nil) {
let store = resolvedDefaults(defaults)
store.set(false, forKey: FlowSessionKeys.flowSessionActive)
store.removeObject(forKey: FlowSessionKeys.flowSessionExpires)
store.removeObject(forKey: FlowSessionKeys.flowHeartbeat)
store.removeObject(forKey: FlowSessionKeys.keyboardRecordingState)
store.removeObject(forKey: FlowSessionKeys.transcriptionLanguage)
clearTranscription(defaults: store)
store.removeObject(forKey: FlowSessionKeys.audioLevels)
flush(store)
}
private static func clearTranscription(defaults: UserDefaults) {
defaults.removeObject(forKey: FlowSessionKeys.transcriptionResult)
defaults.removeObject(forKey: FlowSessionKeys.transcriptionError)
}
}
@@ -0,0 +1,35 @@
// FlowSessionKeys.swift
// OSGKeyboard · Shared
//
// App Group keys for TypeWhisper-style Flow sessions between the
// keyboard extension and the host app (Session Owner).
import Foundation
public enum FlowSessionKeys {
public static let flowSessionActive = "flow.flowSessionActive"
public static let flowSessionExpires = "flow.flowSessionExpires"
public static let flowHeartbeat = "flow.flowHeartbeat"
public static let keyboardRecordingState = "flow.keyboardRecordingState"
public static let transcriptionLanguage = "flow.transcriptionLanguage"
public static let transcriptionResult = "flow.transcriptionResult"
public static let transcriptionError = "flow.transcriptionError"
public static let audioLevels = "flow.audioLevels"
/// Heartbeat older than this implies the host app was killed.
public static let heartbeatStaleInterval: TimeInterval = 3
/// Default Flow session length when started from the keyboard.
public static let defaultSessionDuration: TimeInterval = 480
/// Maximum duration for a single keyboard utterance.
public static let maxUtteranceDuration: TimeInterval = 60
public enum RecordingState: String, Sendable, Equatable {
case idle
case recording
case stopped
case processing
case aborted
}
}
@@ -67,6 +67,8 @@ public final class KeyboardState: ObservableObject {
/// `true` kept on the state object because the UI's status
/// badge still wants a single source of truth to read from.
@Published public var onDeviceSupported: Bool = false
/// Seconds remaining in the current utterance (Flow tap-to-talk).
@Published public var utteranceRemainingSeconds: Int = Int(FlowSessionKeys.maxUtteranceDuration)
/// "local" ASR only, no LLM. "cloud" ASR + optional LLM polish.
@Published public var engineMode: String = "cloud"
@@ -0,0 +1,506 @@
// LiveDictationController.swift
// OSGKeyboard · Shared
//
// Unified on-device dictation session: mic capture + iOS 26 SpeechAnalyzer.
// Used by the keyboard preview sheet, host-app dictation handoff, and any
// other foreground surface that needs live ASR without duplicating pipeline code.
// Owns its own AVAudioEngine + AVAudioSession, downsamples to 16 kHz
// mono Float32 on the audio thread (same as `AudioCaptureService`), and
// feeds `AudioBufferSnapshot` to the shared `ASRService` (the same
// pipeline the real keyboard extension
// uses, so the preview exercises the *real* iOS speech APIs, not a
// stub). Without this the in-app preview was a hardcoded transcript
// and "did you actually call SFSpeechRecognizer?" was a fair review
// note.
//
// Why not reuse `AudioCaptureService` from the extension? It lives in
// `OSGKeyboardExt`, an `app-extension` target the main app can't
// import its symbols. We could move it to `OSGKeyboardShared`, but
// `AVAudioSession` lifecycle differs enough between a keyboard
// extension (no background, no recording entitlement surprise) and a
// foreground app that a copy here is the lesser evil.
import Foundation
import AVFoundation
import Speech
import os
/// Thread-safe relay so the AVAudioEngine tap can yield snapshots without
/// hopping through `@MainActor` (which adds latency and can reorder frames).
private final class CaptureStreamRelay: @unchecked Sendable {
private let lock = OSAllocatedUnfairLock()
private var continuation: AsyncStream<AudioBufferSnapshot>.Continuation?
func bind(_ continuation: AsyncStream<AudioBufferSnapshot>.Continuation) {
lock.withLock { self.continuation = continuation }
}
func yield(_ snapshot: AudioBufferSnapshot) {
lock.withLock { continuation?.yield(snapshot) }
}
func finish() {
lock.withLock {
continuation?.finish()
continuation = nil
}
}
}
@MainActor
public final class LiveDictationController: ObservableObject {
public enum Phase: Equatable {
case idle
case requestingPermission
case recording
case processing
case denied(String)
case error(String)
}
@Published public private(set) var phase: Phase = .idle
/// Normalized 0...1 RMS for the disc level meter. Polled from the
/// audio tap via `Task { @MainActor in ... }` the tap itself
/// runs on a real-time audio thread, so we never touch published
/// state from there.
@Published public private(set) var level: Double = 0
@Published public private(set) var currentPartial: String = ""
@Published public private(set) var errorMessage: String?
/// Set when a `.final` ASR event lands. The owning sheet observes
/// this and appends the text to its textbox, then clears it so the
/// next recording starts from zero.
@Published public var lastFinal: String = ""
private let asr: ASRService = ASRServiceFactory.make()
private let audioEngine = AVAudioEngine()
/// `internal` (not `private`) so the regression test in
/// `OSGKeyboardTests/PreviewASRControllerStateTests.swift` can
/// install a known consumer task and assert `stop()` doesn't
/// cancel it. The class is `@MainActor`-isolated, so the
/// natural Swift 6 isolation rules still prevent production
/// code outside the class from racing on it.
public var asrTask: Task<Void, Never>?
private let streamRelay = CaptureStreamRelay()
private var didConfigureAudioSession = false
private var didInstallTap = false
public init() {}
/// Start dictation using a persisted settings locale id (`auto`, `zh-Hans`, ).
public func start(localeId: String) async {
await start(locale: SpeechLocaleResolver.resolve(localeId))
}
public func start(locale: Locale) async {
// Re-entry guard: ignore taps that arrive while we're already
// running. (The sheet's `cyclePhase` is also guarded, but
// async race windows are easier to lock down here.)
switch phase {
case .recording, .requestingPermission, .processing:
return
default:
break
}
// Cancel any leftover consumer task from a previous recording.
// Normally `stop()` lets the task run to completion (so it can
// see the `.final` and transition out of `.processing`), but if
// the user smashed the disc twice stop, then immediately
// start the previous task might still be draining. Cancel it
// here so we don't have two consumer tasks fighting over the
// same `events` stream.
asrTask?.cancel()
asrTask = nil
teardownCapturePipeline()
phase = .requestingPermission
currentPartial = ""
lastFinal = ""
errorMessage = nil
level = 0
// 1. Microphone permission. The helper is `nonisolated` so the
// (iOS < 17) callback closure does not inherit `@MainActor`
// `AVAudioSession.requestRecordPermission` delivers on a TCC
// reply queue, and a `@MainActor`-inferred closure body there
// hits `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift`.
let micGranted = await Self.requestMicrophonePermission()
guard micGranted else {
phase = .denied(NSLocalizedString("keyboard.denied.mic", comment: ""))
return
}
// 2. Speech recognition permission. Same reasoning as above:
// the callback fires on TCC's reply queue, NOT the main queue.
let speechGranted = await Self.requestSpeechRecognitionPermission()
guard speechGranted else {
phase = .denied(NSLocalizedString("keyboard.denied.speech", comment: ""))
return
}
// 3. Audio session only configure once per process.
//
// Category is `.record` (not `.playAndRecord`) because the
// preview never plays back audio it just records from the
// mic and hands the buffers to `SpeechAnalyzer`. On the
// iOS Simulator, `.playAndRecord` requires the
// `AURemoteIO` Audio Unit's *output* side to also be
// enabled, but the simulator's "speaker" reports a 0 Hz
// hardware format, so `AURemoteIO::enable` fails with
// `kAudioUnitErr_FormatNotSupported` (-10851) and any
// subsequent `installTap` traps with "Failed to create tap
// due to format mismatch". `.record` skips the output
// side entirely, so the simulator can record.
//
// The real keyboard extension (`OSGKeyboardExt`) keeps
// `.playAndRecord` because it runs on a real device where
// the output side has a real hardware format, and may want
// to play click sounds / haptic feedback. Only the preview
// needs the simulator-friendly category.
if !didConfigureAudioSession {
do {
let session = AVAudioSession.sharedInstance()
try session.setCategory(.record,
mode: .measurement,
options: [])
try session.setActive(true, options: .notifyOthersOnDeactivation)
didConfigureAudioSession = true
} catch {
debug("audio session failed: \(error.localizedDescription)")
phase = .error(String.localizedStringWithFormat(
NSLocalizedString("preview.error.audioSession", comment: ""),
error.localizedDescription
))
return
}
}
// 4. Spin up the engine + ASR.
phase = .recording
startEngineAndASR(locale: locale)
}
public func stop() {
// Don't `asrTask?.cancel()` here see the comment in
// `startEngineAndASR` for the full rationale. Short version:
// cancelling the consumer task at the same moment we close the
// audio stream also triggers the producer's
// `continuation.onTermination self?.cancel()` cascade, which
// marks the producer's outer task as cancelled and skips the
// `.final` event. The UI is then left in `.processing` forever
// because no one schedules the transition out. The consumer
// task naturally exits when `events` finishes, so the right
// thing is to let it run.
//
// If a previous `asrTask` is somehow still running (e.g. the
// user smashed the disc twice quickly), `start()` cancels it
// at the entry point as a safety net.
teardownCapturePipeline()
// Fallback: if we already have a meaningful partial but the
// backend never emits `.final`, promote the partial so the
// preview still inserts text after "".
let partial = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
if !partial.isEmpty && lastFinal.isEmpty {
lastFinal = partial
currentPartial = ""
}
if phase == .recording {
phase = .processing
}
// Deactivate so the user's music resumes if the preview is
// dismissed mid-recording.
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
// Safety net: if the ASR pipeline never produces a `.final`
// (analyzer hang, system glitch, dropped continuation), force
// the UI back to idle after a short delay so the user isn't
// stuck. Normal recordings complete well under a second, so
// the 3-second budget is only hit on the unhappy path; if the
// pipeline finishes first and flips the phase to `.idle` (or
// `.error`), the check below no-ops.
Task { @MainActor [weak self] in
try? await Task.sleep(for: .seconds(3))
guard let self else { return }
if self.phase == .processing {
let stalePartial = self.currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
if !stalePartial.isEmpty, self.lastFinal.isEmpty {
self.debug("processing timeout, using partial")
self.lastFinal = stalePartial
self.currentPartial = ""
}
self.phase = .idle
}
}
}
public func reset() {
// Called by the sheet after appending `lastFinal` to the textbox,
// so the next recording can produce a fresh final without us
// double-appending.
lastFinal = ""
if phase == .processing {
phase = .idle
}
}
// MARK: - Engine + ASR
private func startEngineAndASR(locale: Locale) {
let inputNode = audioEngine.inputNode
let hwFormat = inputNode.outputFormat(forBus: 0)
// Pre-flight check: a placeholder / unconfigured input bus
// reports `sampleRate == 0` (or `channelCount == 0`).
// `installTap` on such a bus traps with "Failed to create
// tap due to format mismatch" (an NSException, not a Swift
// `Error`, so we can't `try`/`catch` it). The safest fix
// is to refuse the tap up front and surface a clear
// `.error` phase instead of crashing the app. We've seen
// this on the iOS Simulator when the host's microphone
// permission isn't granted to CoreSimulator, and on
// devices where the audio session is in an unexpected
// state from a previous foreground/background transition.
guard hwFormat.sampleRate > 0, hwFormat.channelCount > 0 else {
debug("invalid hardware format sr=\(hwFormat.sampleRate) ch=\(hwFormat.channelCount)")
phase = .error(
String.localizedStringWithFormat(
NSLocalizedString("preview.error.micUnavailable", comment: ""),
hwFormat.sampleRate,
Int(hwFormat.channelCount)
)
)
return
}
let targetSampleRate: Double = 16_000
guard let targetFormat = AVAudioFormat(
commonFormat: .pcmFormatFloat32,
sampleRate: targetSampleRate,
channels: 1,
interleaved: false
) else {
phase = .error(NSLocalizedString("preview.error.formatCreate", comment: ""))
return
}
guard let converter = AVAudioConverter(from: hwFormat, to: targetFormat) else {
debug("converter creation failed")
phase = .error(NSLocalizedString("preview.error.converterCreate", comment: ""))
return
}
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
streamRelay.bind(continuation)
// Tap the hardware input. The closure passed to `installTap` runs
// on the AVAudioEngine real-time audio thread. In Swift 6 strict
// concurrency, a closure literal defined inside a `@MainActor`
// method inherits `@MainActor` isolation, which would trip
// `dispatch_assert_queue_fail` on first invocation from the
// audio thread. The fix is to build the actual tap body in a
// `nonisolated` helper (`makeAudioTapBlock`) and have the
// installTap closure be a single function reference function
// references never carry inferred isolation, so the dispatch
// runtime is happy and the body runs wherever AVAudioEngine
// wants it (the audio thread).
let onMeter: @Sendable (Double) -> Void = { [weak self] meter in
Task { @MainActor [weak self] in
guard let self else { return }
// Lightweight smoothing so the disc ring doesn't jitter.
self.level = self.level * 0.55 + meter * 0.45
}
}
let relay = streamRelay
let onSnapshot: @Sendable (AudioBufferSnapshot) -> Void = { snapshot in
relay.yield(snapshot)
}
let tap = Self.makeAudioTapBlock(
converter: converter,
targetFormat: targetFormat,
hwFormat: hwFormat,
onMeter: onMeter,
onSnapshot: onSnapshot
)
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
didInstallTap = true
audioEngine.prepare()
do {
try audioEngine.start()
} catch {
debug("audio engine start failed: \(error.localizedDescription)")
phase = .error(String.localizedStringWithFormat(
NSLocalizedString("preview.error.engineStart", comment: ""),
error.localizedDescription
))
return
}
// 5. Wire up ASR.
let events = asr.transcribe(
stream: stream,
locale: locale
)
asrTask = Task { @MainActor [weak self] in
guard let self else { return }
for await event in events {
switch event {
case .capability:
break
case .partial(let s):
self.currentPartial = s
case .final(let s):
let trimmed = s.trimmingCharacters(in: .whitespacesAndNewlines)
self.lastFinal = trimmed
self.currentPartial = ""
self.phase = .idle
case .error(let m):
self.debug("asr error: \(m)")
self.teardownCapturePipeline()
self.errorMessage = m
self.phase = .error(m)
}
}
}
}
// MARK: - Permission helpers (nonisolated)
//
// `SFSpeechRecognizer.requestAuthorization` delivers its callback
// on a TCC reply queue, NOT the main queue. If we wrap that
// callback inline in `start(locale:)` which is `@MainActor`
// Swift 6 strict concurrency infers the closure body as
// `@MainActor`, and the runtime crashes on
// `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift` as
// soon as TCC calls us back.
//
// The first attempt (commit `e8a0310`) extracted the entire
// permission request into a `nonisolated static func` helper.
// That worked in isolation, but the Swift 6 optimizer
// inlined those helpers back into `start(locale:)`. After
// inlining, the `withCheckedContinuation` body and the
// `requestAuthorization` callback were re-typed in the
// `@MainActor` context of the caller, and the runtime
// assertion came right back same crash, different symbol:
// `closure #1 in closure #2 in PreviewASRController.start(locale:)`.
//
// The fix that survives inlining is the *function-reference*
// pattern, the same one used for `installTap` in
// `makeAudioTapBlock` below. The callback is built in a
// `nonisolated` static helper that takes a `CheckedContinuation`
// and returns the `(Status) -> Void` handler. The body of that
// helper has no enclosing actor, so the closure is created in
// nonisolated context. When TCC calls us back, the runtime
// sees a nonisolated closure on a non-main queue and is happy.
//
// `cont.resume(...)` is itself thread-safe on
// `CheckedContinuation`, so we don't need to hop back to the
// main actor before resuming.
private nonisolated static func requestMicrophonePermission() async -> Bool {
// iOS 17+ API; the iOS < 17 fallback (`AVAudioSession.recordPermission`
// + `requestRecordPermission` callback) is gone now that the
// deployment target is iOS 26.
switch AVAudioApplication.shared.recordPermission {
case .granted: return true
case .denied: return false
case .undetermined: return await AVAudioApplication.requestRecordPermission()
@unknown default: return false
}
}
private nonisolated static func requestSpeechRecognitionPermission() async -> Bool {
await withCheckedContinuation { (cont: CheckedContinuation<Bool, Never>) in
SFSpeechRecognizer.requestAuthorization(
Self.makeSpeechAuthHandler(continuation: cont)
)
}
}
private nonisolated static func makeSpeechAuthHandler(
continuation: CheckedContinuation<Bool, Never>
) -> @Sendable (SFSpeechRecognizerAuthorizationStatus) -> Void {
return { status in
continuation.resume(returning: status == .authorized)
}
}
// MARK: - Audio tap (nonisolated, runs on AVAudioEngine render thread)
//
// `AVAudioNode.installTap`'s callback fires on the audio engine's
// real-time render thread. In Swift 6 strict concurrency, a closure
// literal defined inside a `@MainActor` method inherits `@MainActor`
// isolation and `dispatch_assert_queue_fail` fires the moment
// the runtime tries to dispatch that closure on a non-main queue.
//
// The trick is to build the actual tap body in a `nonisolated`
// function and have the installTap closure be a *function reference*
// to that helper. Function references never carry inferred
// isolation, so the dispatch runtime is satisfied and the body
// runs wherever AVAudioEngine wants. State updates to
// `self.level` and the AsyncStream continuation hop back to the
// main actor via `Task { @MainActor in }`, which is itself
// safe to call from a non-isolated context.
private nonisolated static func makeAudioTapBlock(
converter: AVAudioConverter,
targetFormat: AVAudioFormat,
hwFormat: AVAudioFormat,
onMeter: @Sendable @escaping (Double) -> Void,
onSnapshot: @Sendable @escaping (AudioBufferSnapshot) -> Void
) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
// `@Sendable` on the returned closure makes the Sendable
// conformance explicit. `AVAudioNodeTapBlock` is declared as
// a plain escaping closure in the SDK; we cast at the call
// site via `as @Sendable`.
return { buffer, _ in
// 1) Level meter from raw hardware buffer.
let n = Int(buffer.frameLength)
var sumSquares: Float = 0
if let channelData = buffer.floatChannelData?[0], n > 0 {
for i in 0..<n {
let v = channelData[i]
sumSquares += v * v
}
}
let rms = n > 0 ? sqrtf(sumSquares / Float(n)) : 0
let meter = min(Double(rms) * 4.0, 1.0)
onMeter(meter)
// 2) Downsample to 16 kHz mono Float32 for ASR (matches
// `AudioCaptureService` and Apple's `considering:` hint).
let outFrames = AVAudioFrameCount(
Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate
)
guard outFrames > 0,
let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outFrames)
else { return }
var error: NSError?
let status = converter.convert(to: outBuffer, error: &error) { _, outStatus in
outStatus.pointee = .haveData
return buffer
}
guard status == .haveData, error == nil, outBuffer.frameLength > 0 else { return }
let snapshot = AudioBufferSnapshot(buffer: outBuffer)
guard !snapshot.samples.isEmpty else { return }
onSnapshot(snapshot)
}
}
private func teardownCapturePipeline() {
if didInstallTap {
audioEngine.inputNode.removeTap(onBus: 0)
didInstallTap = false
}
if audioEngine.isRunning {
audioEngine.stop()
}
streamRelay.finish()
}
private func debug(_ message: String) {
#if DEBUG
print("🎙️[LiveDictationController] \(message)")
#endif
}
}
@@ -0,0 +1,25 @@
// SpeechLocaleResolver.swift
// OSGKeyboard · Shared
//
// Maps persisted `localeId` settings to a `Locale` suitable for
// `DictationTranscriber.supportedLocale(equivalentTo:)`.
import Foundation
public enum SpeechLocaleResolver {
/// Resolve a stored locale id (`auto`, `zh-Hans`, ) for on-device ASR.
public static func resolve(_ localeId: String) -> Locale {
let raw: String
if localeId == "auto" {
raw = Locale.preferredLanguages.first ?? "en-US"
} else {
raw = localeId
}
let normalized = raw.replacingOccurrences(of: "_", with: "-").lowercased()
if normalized.hasPrefix("zh") { return Locale(identifier: "zh-Hans") }
if normalized.hasPrefix("ja") { return Locale(identifier: "ja-JP") }
if normalized.hasPrefix("ko") { return Locale(identifier: "ko-KR") }
if normalized.hasPrefix("en") { return Locale(identifier: "en-US") }
return Locale(identifier: raw.replacingOccurrences(of: "_", with: "-"))
}
}
@@ -0,0 +1,17 @@
// DictationTextComposer.swift
// OSGKeyboard · Shared
//
// Merges pre-dictation anchor text with a live cumulative transcript.
import Foundation
public enum DictationTextComposer {
/// Combine text that existed before dictation with the current live transcript.
public static func compose(anchor: String, live: String) -> String {
let trimmed = live.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return anchor }
if anchor.isEmpty { return trimmed }
if anchor.last == " " || anchor.last == "\n" { return anchor + trimmed }
return anchor + " " + trimmed
}
}
@@ -0,0 +1,59 @@
// DictationBridgeTests.swift
// OSGKeyboardTests
import XCTest
@testable import OSGKeyboardShared
final class DictationBridgeTests: XCTestCase {
private func makeDefaults() -> UserDefaults {
let suite = "group.com.osgkeyboard.shared.tests.dictation.\(UUID().uuidString)"
let defaults = UserDefaults(suiteName: suite)!
defaults.removePersistentDomain(forName: suite)
return defaults
}
func testStoreAndConsumeTranscript() {
let defaults = makeDefaults()
DictationBridge.storePendingTranscript(" hello ", defaults: defaults)
let consumed = DictationBridge.consumePendingTranscript(defaults: defaults)
XCTAssertEqual(consumed, "hello")
XCTAssertNil(DictationBridge.consumePendingTranscript(defaults: defaults))
}
func testConsumeIgnoresExpiredTranscript() {
let defaults = makeDefaults()
DictationBridge.storePendingTranscript("stale", defaults: defaults)
// maxAge = 1ms, then delay to force expiry
usleep(2_000)
let consumed = DictationBridge.consumePendingTranscript(maxAge: 0.001, defaults: defaults)
XCTAssertNil(consumed)
}
func testStatusLifecycle() {
let defaults = makeDefaults()
DictationBridge.markRequested(defaults: defaults)
XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .requested)
DictationBridge.setStatus(.recording, defaults: defaults)
XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .recording)
DictationBridge.storePendingTranscript("ok", defaults: defaults)
XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .done)
_ = DictationBridge.consumePendingTranscript(defaults: defaults)
XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .idle)
}
func testStatusMessageAndTimestamp() {
let defaults = makeDefaults()
DictationBridge.setStatus(.error, message: "fail", defaults: defaults)
let snapshot = DictationBridge.currentStatus(defaults: defaults)
XCTAssertEqual(snapshot.status, .error)
XCTAssertEqual(snapshot.message, "fail")
XCTAssertGreaterThan(snapshot.updatedAt, 0)
}
}
@@ -0,0 +1,52 @@
// FlowSessionBridgeTests.swift
// OSGKeyboardTests
import XCTest
@testable import OSGKeyboardShared
final class FlowSessionBridgeTests: XCTestCase {
private func makeDefaults() -> UserDefaults {
let suite = "group.com.osgkeyboard.shared.tests.flow.\(UUID().uuidString)"
let defaults = UserDefaults(suiteName: suite)!
defaults.removePersistentDomain(forName: suite)
return defaults
}
func testSessionActiveRequiresFreshHeartbeat() {
let defaults = makeDefaults()
FlowSessionBridge.markSessionActive(duration: 60, defaults: defaults)
XCTAssertTrue(FlowSessionBridge.isSessionActive(defaults: defaults))
let staleHeartbeat = Date().timeIntervalSince1970 - 10
defaults.set(staleHeartbeat, forKey: FlowSessionKeys.flowHeartbeat)
XCTAssertFalse(FlowSessionBridge.isSessionActive(defaults: defaults))
}
func testRecordingStateRoundTrip() {
let defaults = makeDefaults()
FlowSessionBridge.setRecordingState(.recording, defaults: defaults)
XCTAssertEqual(FlowSessionBridge.recordingState(defaults: defaults), .recording)
FlowSessionBridge.setRecordingState(.stopped, defaults: defaults)
XCTAssertEqual(FlowSessionBridge.recordingState(defaults: defaults), .stopped)
}
func testConsumeTranscriptionResultClearsKey() {
let defaults = makeDefaults()
FlowSessionBridge.storeTranscriptionResult("hello", defaults: defaults)
XCTAssertEqual(FlowSessionBridge.consumeTranscriptionResult(defaults: defaults), "hello")
XCTAssertNil(FlowSessionBridge.consumeTranscriptionResult(defaults: defaults))
}
func testClearFlowStateRemovesSessionKeys() {
let defaults = makeDefaults()
FlowSessionBridge.markSessionActive(defaults: defaults)
FlowSessionBridge.storeTranscriptionResult("x", defaults: defaults)
FlowSessionBridge.clearFlowState(defaults: defaults)
XCTAssertFalse(defaults.bool(forKey: FlowSessionKeys.flowSessionActive))
XCTAssertNil(FlowSessionBridge.consumeTranscriptionResult(defaults: defaults))
XCTAssertEqual(FlowSessionBridge.recordingState(defaults: defaults), .idle)
}
}
@@ -29,8 +29,7 @@
// the ASR to never yield `.final` and asserts the timeout fires.
import XCTest
@testable import OSGKeyboard
import AVFoundation
@testable import OSGKeyboardShared
@MainActor
final class PreviewASRControllerStateTests: XCTestCase {
@@ -41,7 +40,7 @@ final class PreviewASRControllerStateTests: XCTestCase {
/// is the regression test for the original bug if it ever
/// fails, the disc is stuck again.
func testStopDoesNotCancelConsumerTask() {
let controller = PreviewASRController()
let controller = LiveDictationController()
let consumerTask = Task<Void, Never> {}
controller.asrTask = consumerTask
@@ -58,7 +57,7 @@ final class PreviewASRControllerStateTests: XCTestCase {
/// misbehave no double-cancel, no extra state transitions.
/// This is the contract `.onDisappear` relies on.
func testStopIsIdempotent() {
let controller = PreviewASRController()
let controller = LiveDictationController()
let consumerTask = Task<Void, Never> {}
controller.asrTask = consumerTask
+270
View File
@@ -0,0 +1,270 @@
# OSGKeyboard · TypeWhisper Flow 迁移蓝图与任务追踪
> 目标:把当前“每次按语音都尝试跳转主 App”的模式,迁移为“Flow Session 会话模式”,实现**仅键盘侧连续语音输入**(会话有效期间无需反复跳转)。
>
> 维护方式:每完成一个任务,把对应复选框从 `[ ]` 改为 `[x]`,并填写“完成记录”。
---
## 1) 迁移蓝图(最终目标架构)
### 1.1 第一性目标
- 主链路不依赖每次 `openHostApp(dictate)` 成功。
- 会话有效期间,键盘只做“开始/停止”信号与结果插入。
- 跳转主 App 降级为“会话初始化/修复”路径。
- 任何异常都可恢复,不出现卡死状态。
### 1.2 目标架构
- **主 AppSession Owner**
- 维护 Flow 会话生命周期(active/expired/inactive
- 持续写心跳(heartbeat
- **持有唯一 continuous 音频管线**(见 §1.5
- 处理键盘录音状态信号并执行识别
- 回写 `transcriptionResult/transcriptionError`
- **键盘扩展(Signal + Insert**
- 判断会话是否有效(active + expires + heartbeat
- 会话有效:写 `recording/stopped/aborted`
- 会话无效:引导启动会话(一次性)
- 轮询结果并 `insertText`
- **App Group(单一事实源)**
- 所有跨进程状态仅通过共享键传递
### 1.3 关键共享键
- `flowSessionActive: Bool`
- `flowSessionExpires: TimeInterval`
- `flowHeartbeat: TimeInterval`
- `keyboardRecordingState: String` (`idle|recording|stopped|processing|aborted`)
- `transcriptionLanguage: String`
- `transcriptionResult: String`
- `transcriptionError: String`
- `audioLevels: [Float]`(键盘波形,**Phase 1 必做**
### 1.4 状态机约束
- `recording` 仅在 `flowSessionActive = true` 且会话未过期时生效。
- `stopped` 必须最终归并到 `done|error|idle`
- 任意异常必须显式写 `transcriptionError` 并回到 `idle`
### 1.5 音频不变量(不可违背 — 对齐 TypeWhisper / SwiftSpeak SwiftLink
> **历史教训**:曾用 `.playback` 静音保活 + utterance 时 `LiveDictationController.start()` 临时开麦,导致 `Session activation failed` 与双 engine 崩溃。**禁止再使用该模式。**
| # | 规则 | TypeWhisper 对应 | OSGKeyboard 实现 |
|---|------|------------------|------------------|
| A1 | 会话启动时配置 **`.playAndRecord`**`mode: .measurement`),`setActive(true)` **一次** | `startFlowSession()` | `FlowContinuousCapture.start()` |
| A2 | 立刻 **`startContinuousRecording()`**:一个 `AVAudioEngine` + **常驻** `inputNode` tap | `startContinuousRecording()` | 同上,tap 在 `start()` 安装 |
| A3 | utterance 期间 **禁止** stop/start engine、**禁止** deactivate/reactivate session | `isRecordingAtomic` gating | `isUtteranceActive` + `beginUtterance`/`endUtterance` |
| A4 | 键盘 `recording` → 只 flip 标志 + 启动 ASR consumer`stopped` → 结束 ASR stream + finalize | `checkKeyboardSignal()` | `FlowSessionManager.handleKeyboardSignal()` |
| A5 | **`audioLevels` 从 tap 计算**,主线程写入 App Group**禁止**在 audio realtime 线程 `UserDefaults.synchronize()` | tap 写 levels(我们改为 main 线程 flush 更安全) | `FlowLevelStore` + `startLevelPublishing()` |
| A6 | Flow 期间 **禁止** 调用 `LiveDictationController.start()`(预览 / legacy dictate 专用) | `AudioRecordingService` 拒绝 Flow active | `FlowSessionManager` 直接用 `ASRService` |
| A7 | 会话结束才 `removeTap` / `engine.stop()` / `setActive(false)` | `endFlowSession()` | `FlowSessionManager.endSession()` |
**错误模式(已废弃,勿恢复):**
- ❌ `.playback` + 静音 `AVAudioPlayerNode` 保活
- ❌ utterance 时 stop 保活 engine 再开第二个 engine 录音
- ❌ 复用 `LiveDictationController` 作为 Flow utterance 入口
**参考项目分工:**
- **TypeWhisper**(主参考):`FlowSessionManager` + continuous tap + `SFSpeechAudioBufferRecognitionRequest` / batch
- **SwiftSpeak SwiftLink**(辅参考):Darwin 通知、前台启动约束、streaming/batch 分叉
- **Legacy dictate**(保留):`DictationCaptureView` + `LiveDictationController`,单次 handoff
### 1.6 组件职责
| 组件 | 职责 |
|------|------|
| `FlowContinuousCapture` | 会话级 engine + tap + utterance gating + levels |
| `FlowSessionManager` | 生命周期、轮询、ASR finalize、可选 polish |
| `FlowSessionBridge` | App Group 读写 |
| `LiveDictationController` | 主 App 预览、legacy `dictate`**不用于 Flow** |
| `KeyboardViewController` | 信号 + 轮询 + insertText |
---
## 2) 详细改动清单(按文件)
## Phase 1 · 基础设施(会话能力)
### `OSGKeyboardShared/Services/FlowSessionBridge.swift`
- [x] Flow 键读写封装(active/expires/heartbeat/recordingState/result/error
- [x] `storeAudioLevels` / `clearPendingTranscription`
- [x] 保留并兼容现有 `DictationBridge` pending transcript 接口
- [x] `clearFlowState()`
### `OSGKeyboardShared/Services/FlowContinuousCapture.swift`(新增)
- [x] `.playAndRecord` + 常驻 input tap
- [x] utterance gating → `AsyncStream<AudioBufferSnapshot>`
- [x] `FlowLevelStore`(audio 线程写、main 线程读)
### `OSGKeyboard/Services/FlowSessionManager.swift`
- [x] 会话生命周期 + heartbeat + 过期
- [x] 轮询 `keyboardRecordingState` → utterance gating → `ASRService`
- [x] 回写 `transcriptionResult/transcriptionError`
- [x] 主线程发布 `audioLevels`
- [x] **不再**使用 playback keep-alive / `LiveDictationController` for Flow
### `OSGKeyboard/Info.plist``project.yml`
- [x] `UIBackgroundModes: audio`
- [x] 麦克风 / 语音识别权限说明
---
## Phase 2 · 键盘主链路切换
### `OSGKeyboardExt/KeyboardViewController.swift`
- [x] `pressBegan()` 会话判断分流
- [x] 会话有效:写 `recording`;无效:`openHostApp(startflow)` 或提示
- [x] `pressEnded()``stopped`
- [x] 结果轮询 + `insertText`
- [ ] `KeyboardRootView` 会话 UI 细化(可选)
---
## Phase 3 · 体验与稳定性强化
### 主 App / 键盘协同
- [x] 心跳超时判死(`FlowSessionBridge.isSessionActive`
- [x] 识别超时保护(30s finalize
- [x] `audioLevels` 共享
- [ ] Darwin 通知(SwiftSpeak 模式,降低轮询延迟)
- [ ] 会话死掉后一键重启 UI
- [ ] `AudioRouteCoordinator`(蓝牙/路由切换)
### 测试
- [x] App Group 状态机单测
- [ ] Flow continuous capture 单测(需 device / mock
- [ ] 端到端真机回归
---
## 3) 任务追踪面板
## A. 已完成
- [x] A1–A4:架构研究、蓝图、追踪文档
- [x] B1Phase 1 Flow 基础设施(含音频层修正)
- [x] B2:Phase 2 键盘主链路(核心路径)
## B. 待执行
- [ ] B3Phase 3 体验增强(Darwin、路由、恢复 UI)→ 见 §7.7 批次 F
- [ ] B4:端到端真机回归 → 见 §7.7;本地/在线主路径已通过
- [ ] **Phase 4**:§7.27.7 批次 AF
---
## 5) 验收标准(Definition of Done
### Flow 核心(Phase 12
- [x] 本地 / 云端模式均可回填(真机已验证)
- [ ] 连续 20 次语音输入,会话有效期间 **无需** 反复跳主 App
- [ ] Console **无** `Session activation failed` / playback↔record 循环
- [ ] 键盘波形随说话变化(`audioLevels` 非零)
- [ ] 微信/备忘录/Safari 稳定回填
### Phase 4 新增
- [ ] 打开 App 后 **自动** 语音会话(权限齐全时)
- [ ] 杀 App 再开 → **冷启动恢复**(未过期)
- [ ] 键盘 **点按** 开始/结束;60s 倒计时 + 最后 10s 变红
- [ ] Onboarding **分步权限** 完整可走通
- [ ] 隐私政策 URL 可访问;App 内可打开
- [ ] App Store 隐私标签与政策一致
- [ ] 单测覆盖核心状态迁移
---
## 6) 真机验证清单(最小)
1. 打开 App → **自动**进入语音会话(或 Home 卡片显示「进行中」)
2. 切备忘录 → **点按**键盘麦开始 → 再点结束 → **文字出现**
3. **不跳主 App**,重复 5 次
4. 本地模式 + 云端模式各测 1 次
5. 60s 倒计时 + 最后 10 秒变红;到点自动识别
6. 杀 App 再开 → 会话恢复(未过期时)
7. 预览 sheet「开始/停止录音」仍可用(Flow 未启动时)
---
## 7) Phase 4 · 产品体验与上架合规(2026-06 拍板)
> 以下为用户/产品确认规格。**实现顺序建议:B → C → D → A → E → F。**
### 7.1 已锁定决策
| 主题 | 决定 |
|------|------|
| 麦克风交互 | **点按开始 / 再点结束**;识别中不可取消 |
| 会话未启动 | 保持现有逻辑(拉 App / 提示去主 App) |
| 权限引导 | 欢迎 → 麦克风 → 语音识别 → 键盘+完全访问 → 引擎/API;**仅首次或权限未定时** |
| 自动开语音会话 | 进 App 且权限齐全 → **自动开**;有效会话 → **续期****不设关闭开关** |
| 冷启动恢复 | 参考 TypeWhisper `checkExistingSession` |
| Home 按钮 | 自动开后 **隐藏「启动」**;**保留「结束」**;失败显示原因 + 去设置 |
| 单次录音上限 | **60s**;键盘到点 auto `stopped`;倒计时 **A(剩余)+ C(最后 10s 变红)**,显示在按钮内 |
| 隐私政策 URL | **GitHub Pages**(仓库站点,如 `…/privacy` |
| 自动开会话开关 | **先不加** |
### 7.2 批次 A · App Store 合规(上架前必做)
- [x] **A1** 隐私政策页面(GitHub Pages `privacy.html` / `docs/privacy`),en + zh-Hans
- [x] **A2** App 内入口:设置页 + Onboarding 底部「隐私政策」链接
- [ ] **A3** App Store Connect「App 隐私」问卷与政策一致
- [ ] **A4** 复核 / 更新 `PrivacyInfo.xcprivacy`(主 App + 键盘扩展;麦克风、UserDefaults 等)
- [x] **A5** 更新 `NSMicrophoneUsageDescription` / `NSSpeechRecognitionUsageDescription`(含 Flow 自动会话说明)
- [x] **A6** 「完全访问」专项说明(Onboarding + 设置:用途、不上传击键)
- [ ] **A7** 云端模式披露:润色时仅文字发往用户配置的 API
- [ ] **A8**(可选)支持邮箱 / 用户协议
### 7.3 批次 B · 分步权限引导
- [x] **B1** Onboarding 麦克风页(说明 + 按钮触发系统弹窗)
- [x] **B2** Onboarding 语音识别页
- [x] **B3** 键盘 + 「允许完全访问」图文 + 跳转系统设置
- [x] **B4**`PermissionPrimer` 合并;仅首次 / 权限未定时展示
- [x] **B5** 权限被拒降级页(去设置)
### 7.4 批次 C · 语音会话自动化
- [x] **C1** 进 App + 权限 OK → 自动 `startSession``OSGKeyboardApp` / Home
- [x] **C2** 冷启动 `checkExistingSession``FlowSessionManager` init
- [x] **C3** Home 卡片 UX:进行中 + 结束;失败原因 + 去设置;隐藏手动「启动」
- [x] **C4**`scenePhase.active` 续期对齐
### 7.5 批次 D · 键盘点按录音 + 60s 倒计时
- [x] **D1** `RecordButton` 改为 toggle(替换长按手势)
- [x] **D2** 识别中禁用按钮
- [x] **D3** 按钮内剩余时间倒计时(`M:SS`
- [x] **D4** 最后 10 秒变红/橙(A+C
- [x] **D5** 60s 到点自动 `stopped` → 「识别中…」
- [x] **D6** 无障碍 / 占位文案改为「点按说话」类
### 7.6 批次 E · 多语言完善
- [ ] **E1** 键盘扩展:移除 `KeyboardL10n` 硬编码,统一 `Localizable.strings`
- [ ] **E2** `KeyboardViewController` 硬编码中文迁入 strings
- [ ] **E3** 批次 B/C/D/A 新增文案 en + zh-Hans 成对
### 7.7 批次 F · Phase 3 收尾
- [ ] **F1** B3:会话过期键盘提示、Darwin(可选)、路由(可选)
- [ ] **F2** B4:全场景回归(含自动开、60s、杀 App 恢复)
- [ ] **F3** 更新 §5 验收勾选
### 7.8 任务追踪(Phase 4
- [x] P4-0:产品规格拍板(交互、权限、自动会话、60s、隐私 URL)
- [ ] P4-A:上架合规批次(A1/A2/A5 代码侧已完成;A3/A4 待人工)
- [x] P4-B:权限引导
- [x] P4-C:会话自动化
- [x] P4-D:键盘点按 + 倒计时
- [x] P4-E:多语言(核心文案;KeyboardL10n 硬编码待全量迁移)
- [ ] P4-FPhase 3 收尾 + B4 回归
---
## 4) 完成记录
| 日期 | 任务ID | 变更摘要 | 状态 |
|---|---|---|---|
| 2026-06-19 | A1-A4 | 架构研究、方案选择、蓝图与追踪文档 | Done |
| 2026-06-19 | B1-B2 | Flow IPC + 键盘链路;修正 continuous capture 音频层 | Done |
| 2026-06-19 | B4-partial | 真机:本地 + 在线 Flow 可用 | Done |
| 2026-06-19 | P4-0 | Phase 4 产品规格与合规任务清单拍板 | Done |
View File
+149
View File
@@ -0,0 +1,149 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>OSGKeyboard</title>
<meta name="description" content="OSGKeyboard — voice dictation keyboard for iOS with on-device ASR and optional LLM polish.">
<style>
:root {
color-scheme: light dark;
--bg: #f7f7f7;
--card: #fff;
--text: #1a1a1a;
--muted: #666;
--accent: #0a7a55;
--border: #e5e5e5;
}
@media (prefers-color-scheme: dark) {
:root {
--bg: #111;
--card: #1c1c1e;
--text: #f5f5f7;
--muted: #a1a1a6;
--accent: #34c759;
--border: #333;
}
}
* { box-sizing: border-box; }
body {
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
line-height: 1.6;
margin: 0;
background: var(--bg);
color: var(--text);
}
.wrap {
max-width: 720px;
margin: 0 auto;
padding: 2.5rem 1.25rem 3rem;
}
header { margin-bottom: 2rem; }
h1 { font-size: 2rem; line-height: 1.2; margin: 0 0 0.5rem; }
.tagline { color: var(--muted); margin: 0; font-size: 1.05rem; }
.lang { font-size: 0.9rem; margin-top: 0.75rem; }
.lang a { color: var(--accent); }
.card {
background: var(--card);
border: 1px solid var(--border);
border-radius: 14px;
padding: 1.25rem 1.35rem;
margin-bottom: 1rem;
}
.card h2 { font-size: 1.05rem; margin: 0 0 0.5rem; }
.card p { margin: 0; color: var(--muted); }
.links { display: flex; flex-wrap: wrap; gap: 0.75rem; margin-top: 1.25rem; }
.btn {
display: inline-block;
padding: 0.65rem 1rem;
border-radius: 10px;
text-decoration: none;
font-weight: 600;
font-size: 0.95rem;
}
.btn-primary { background: var(--accent); color: #fff; }
.btn-secondary {
background: transparent;
color: var(--accent);
border: 1px solid var(--accent);
}
footer {
margin-top: 2rem;
font-size: 0.85rem;
color: var(--muted);
}
hr { border: none; border-top: 1px solid var(--border); margin: 2rem 0; }
</style>
</head>
<body>
<div class="wrap">
<header>
<h1>OSGKeyboard</h1>
<p class="tagline">Tap to talk. Polished text in any app.</p>
<p class="lang"><a href="#zh">中文</a></p>
</header>
<div class="card">
<h2>Voice dictation keyboard for iOS</h2>
<p>
OSGKeyboard is a custom keyboard extension that transcribes your voice on-device
and optionally polishes the result through an LLM you configure.
Works in WeChat, Notes, Mail, ChatGPT, and anywhere a keyboard appears.
</p>
<div class="links">
<a class="btn btn-primary" href="https://github.com/hkgood/OSGKeyboard">View on GitHub</a>
<a class="btn btn-secondary" href="privacy/">Privacy Policy</a>
</div>
</div>
<div class="card">
<h2>Privacy at a glance</h2>
<p>
Audio is processed on your device for transcription. We do not log ordinary keystrokes.
In Cloud polish mode, only the transcribed text (not audio) is sent to your chosen API provider.
</p>
<div class="links">
<a class="btn btn-secondary" href="privacy/">Read full policy</a>
</div>
</div>
<footer>
<p>© OSGKeyboard · <a href="https://github.com/hkgood/OSGKeyboard">Source code</a></p>
</footer>
<hr id="zh">
<header>
<h1>OSGKeyboard</h1>
<p class="tagline">点按说话,任意 App 里获得润色文字。</p>
</header>
<div class="card">
<h2>iOS 语音输入键盘</h2>
<p>
OSGKeyboard 是一款自定义键盘扩展:在设备端转写语音,并可选通过你配置的 LLM 润色结果。
适用于微信、备忘录、邮件、ChatGPT 等所有出现键盘的场景。
</p>
<div class="links">
<a class="btn btn-primary" href="https://github.com/hkgood/OSGKeyboard">GitHub 仓库</a>
<a class="btn btn-secondary" href="privacy/#zh">隐私政策</a>
</div>
</div>
<div class="card">
<h2>隐私摘要</h2>
<p>
音频在设备端转写。我们不会记录普通击键内容。
云端润色模式下,仅发送转写文字(非音频)到你选择的 API 服务商。
</p>
<div class="links">
<a class="btn btn-secondary" href="privacy/#zh">查看完整政策</a>
</div>
</div>
<footer>
<p>© OSGKeyboard · <a href="https://github.com/hkgood/OSGKeyboard">源代码</a></p>
</footer>
</div>
</body>
</html>
+88
View File
@@ -0,0 +1,88 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>OSGKeyboard Privacy Policy</title>
<style>
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; line-height: 1.6; max-width: 720px; margin: 2rem auto; padding: 0 1rem; color: #1a1a1a; }
h1, h2 { line-height: 1.3; }
a { color: #0a7; }
hr { margin: 2rem 0; border: none; border-top: 1px solid #ddd; }
.lang { font-size: 0.9rem; color: #666; }
</style>
</head>
<body>
<p class="lang"><a href="#zh">中文</a></p>
<h1>OSGKeyboard Privacy Policy</h1>
<p><strong>Last updated:</strong> June 19, 2026</p>
<p>OSGKeyboard is a custom iOS keyboard that turns your voice into text. This policy explains what data the app processes and how it is used.</p>
<h2>What we collect</h2>
<ul>
<li><strong>Voice audio</strong> — captured only while you actively record. On-device mode transcribes locally with Apples speech APIs; raw audio is not uploaded by OSGKeyboard.</li>
<li><strong>Transcribed text</strong> — in Cloud polish mode, the final text (not audio) may be sent to the LLM provider you configure (e.g. OpenAI) for punctuation and formatting.</li>
<li><strong>API credentials</strong> — stored in the iOS Keychain on your device and shared only between the main app and keyboard extension via an App Group.</li>
<li><strong>App preferences</strong> — engine mode, language, and keyboard settings stored in App Group UserDefaults on your device.</li>
</ul>
<h2>What we do not collect</h2>
<ul>
<li>We do <strong>not</strong> log or upload ordinary keystrokes you type with the keyboard.</li>
<li>We do <strong>not</strong> operate analytics or advertising SDKs.</li>
<li>We do <strong>not</strong> sell personal data.</li>
</ul>
<h2>Permissions</h2>
<ul>
<li><strong>Microphone</strong> — required for voice input and background voice sessions.</li>
<li><strong>Speech recognition</strong> — required for on-device transcription.</li>
<li><strong>Full Access</strong> — required so the keyboard can reach the microphone, read your API key, and communicate with the main app. Full Access does not grant us access to everything you type; we do not exfiltrate keystrokes.</li>
</ul>
<h2>Third parties</h2>
<p>When you choose Cloud polish mode, transcribed text is sent to the API endpoint you configure. That providers privacy policy applies to those requests.</p>
<h2>Data retention</h2>
<p>Settings and API keys remain on your device until you delete the app or reset settings. Transcription results are passed to the host app you are typing in and are not stored long-term by OSGKeyboard.</p>
<h2>Contact</h2>
<p>Questions: open an issue at <a href="https://github.com/hkgood/OSGKeyboard">github.com/hkgood/OSGKeyboard</a>.</p>
<hr id="zh">
<h1>OSGKeyboard 隐私政策</h1>
<p><strong>更新日期:</strong>2026 年 6 月 19 日</p>
<p>OSGKeyboard 是一款 iOS 自定义键盘,可将语音转为文字。本政策说明应用处理哪些数据及用途。</p>
<h2>我们处理的数据</h2>
<ul>
<li><strong>语音音频</strong> — 仅在你主动录音时采集。本地模式在设备端通过 Apple 语音识别转写,OSGKeyboard 不会上传原始录音。</li>
<li><strong>转写文字</strong> — 云端润色模式下,最终文字(非音频)可能发送到你配置的 LLM 服务商以整理标点和格式。</li>
<li><strong>API 凭证</strong> — 保存在设备 Keychain,仅通过 App Group 在主 App 与键盘扩展间共享。</li>
<li><strong>应用偏好</strong> — 引擎、语言等设置保存在设备 App Group 中。</li>
</ul>
<h2>我们不收集的内容</h2>
<ul>
<li>我们<strong>不会</strong>记录或上传你平时在键盘上的击键内容。</li>
<li>我们<strong>不会</strong>集成广告或第三方分析 SDK。</li>
<li>我们<strong>不会</strong>出售个人数据。</li>
</ul>
<h2>权限说明</h2>
<ul>
<li><strong>麦克风</strong> — 语音输入与后台语音会话所需。</li>
<li><strong>语音识别</strong> — 端侧转写所需。</li>
<li><strong>完全访问</strong> — 使键盘能使用麦克风、读取 API Key 并与主 App 通信。完全访问不代表我们会收集全部击键内容。</li>
</ul>
<h2>第三方</h2>
<p>选择云端润色时,转写文字会发往你配置的 API,该服务商的隐私政策适用于相关请求。</p>
<h2>数据保留</h2>
<p>设置与 API Key 保留在设备上,直至卸载或重置。识别结果写入你正在使用的宿主 App,OSGKeyboard 不会长期存储。</p>
<h2>联系</h2>
<p>问题反馈:<a href="https://github.com/hkgood/OSGKeyboard">github.com/hkgood/OSGKeyboard</a></p>
</body>
</html>
+13 -10
View File
@@ -55,8 +55,8 @@ targets:
# each request. The first entry below becomes each process's
# default access group, so the Keychain helper does not need to
# specify kSecAttrAccessGroup explicitly.
com.apple.security.keychain-access-groups:
- com.osgkeyboard.shared
keychain-access-groups:
- $(AppIdentifierPrefix)com.osgkeyboard.shared
resources:
- path: OSGKeyboard/Assets.xcassets
- path: OSGKeyboard/en.lproj
@@ -80,8 +80,10 @@ targets:
- UIInterfaceOrientationPortrait
UIApplicationSceneManifest:
UIApplicationSupportsMultipleScenes: false
NSMicrophoneUsageDescription: "OSGKeyboard needs microphone access to transcribe your voice into text."
NSSpeechRecognitionUsageDescription: "OSGKeyboard uses on-device speech recognition to transcribe your dictation. Audio never leaves your device."
NSMicrophoneUsageDescription: "OSGKeyboard uses the microphone for voice dictation and keeps a background audio session active while a voice session is running."
NSSpeechRecognitionUsageDescription: "OSGKeyboard uses on-device speech recognition to transcribe your voice. Audio is processed on your device and is not uploaded for transcription."
UIBackgroundModes:
- audio
NSAppTransportSecurity:
NSAllowsArbitraryLoads: false
CFBundleURLTypes:
@@ -111,8 +113,9 @@ targets:
excludes:
- "en.lproj"
- "zh-Hans.lproj"
- path: OSGKeyboardExt/en.lproj
- path: OSGKeyboardExt/zh-Hans.lproj
resources:
- path: OSGKeyboardExt/en.lproj/Localizable.strings
- path: OSGKeyboardExt/zh-Hans.lproj/Localizable.strings
settings:
base:
IPHONEOS_DEPLOYMENT_TARGET: "26.0"
@@ -124,16 +127,16 @@ targets:
# Shared with the host app so the keyboard extension can read the
# user's LLM API key. See OSGKeyboard/OSGKeyboard.entitlements
# for the full rationale.
com.apple.security.keychain-access-groups:
- com.osgkeyboard.shared
keychain-access-groups:
- $(AppIdentifierPrefix)com.osgkeyboard.shared
info:
path: OSGKeyboardExt/Info.plist
properties:
CFBundleDisplayName: OSGKeyboard
CFBundleShortVersionString: "$(MARKETING_VERSION)"
CFBundleVersion: "$(CURRENT_PROJECT_VERSION)"
NSMicrophoneUsageDescription: "OSGKeyboard needs microphone access to transcribe your voice into text."
NSSpeechRecognitionUsageDescription: "OSGKeyboard uses on-device speech recognition to transcribe your dictation. Audio never leaves your device."
NSMicrophoneUsageDescription: "OSGKeyboard uses the microphone for voice dictation and keeps a background audio session active while a voice session is running."
NSSpeechRecognitionUsageDescription: "OSGKeyboard uses on-device speech recognition to transcribe your voice. Audio is processed on your device and is not uploaded for transcription."
NSExtension:
NSExtensionAttributes:
IsASCIICapable: false