feat: TypeWhisper Flow sessions, Phase 4 UX, and GitHub Pages privacy site

Migrate keyboard dictation to continuous Flow sessions with auto-start,
tap-to-toggle recording, 60s countdown, five-step onboarding, and App Group
IPC. Add docs/ GitHub Pages site with en/zh privacy policy for App Store compliance.
This commit is contained in:
Rocky
2026-06-19 18:05:50 +08:00
parent 6148d05093
commit 7f059dbd45
48 changed files with 3815 additions and 1065 deletions
Binary file not shown.

Before

Width:  |  Height:  |  Size: 44 KiB

After

Width:  |  Height:  |  Size: 39 KiB

+6 -2
View File
@@ -42,14 +42,18 @@
<false/>
</dict>
<key>NSMicrophoneUsageDescription</key>
<string>OSGKeyboard needs microphone access to transcribe your voice into text.</string>
<string>OSGKeyboard uses the microphone for voice dictation and keeps a background audio session active while a voice session is running.</string>
<key>NSSpeechRecognitionUsageDescription</key>
<string>OSGKeyboard uses on-device speech recognition to transcribe your dictation. Audio never leaves your device.</string>
<string>OSGKeyboard uses on-device speech recognition to transcribe your voice. Audio is processed on your device and is not uploaded for transcription.</string>
<key>UIApplicationSceneManifest</key>
<dict>
<key>UIApplicationSupportsMultipleScenes</key>
<false/>
</dict>
<key>UIBackgroundModes</key>
<array>
<string>audio</string>
</array>
<key>UILaunchScreen</key>
<dict>
<key>UIColorName</key>
+2 -2
View File
@@ -8,9 +8,9 @@
</array>
<key>com.apple.security.device.audio-input</key>
<true/>
<key>com.apple.security.keychain-access-groups</key>
<key>keychain-access-groups</key>
<array>
<string>com.osgkeyboard.shared</string>
<string>$(AppIdentifierPrefix)com.osgkeyboard.shared</string>
</array>
</dict>
</plist>
+33 -2
View File
@@ -6,14 +6,19 @@ import OSGKeyboardShared
@main
struct OSGKeyboardApp: App {
@Environment(\.scenePhase) private var scenePhase
@StateObject private var config = ProviderConfig.shared
@StateObject private var dictationCoordinator = DictationSessionCoordinator()
@StateObject private var flowManager = FlowSessionManager()
var body: some Scene {
WindowGroup {
ThemedRoot {
if AppGroup.isAvailable {
if config.isConfigured {
if config.hasCompletedOnboarding {
HomeView()
.onAppear { flowManager.autoStartIfNeeded() }
} else {
OnboardingView(config: config)
}
@@ -21,6 +26,32 @@ struct OSGKeyboardApp: App {
AppGroupErrorView()
}
}
.environmentObject(flowManager)
.onOpenURL { url in
guard url.scheme == "osgkeyboard" else { return }
switch url.host {
case "dictate":
dictationCoordinator.present()
case "startflow":
flowManager.startSession()
default:
break
}
}
.fullScreenCover(isPresented: $dictationCoordinator.isPresenting) {
DictationCaptureView(
config: config,
coordinator: dictationCoordinator
)
}
.onChange(of: scenePhase) { _, phase in
guard phase == .active, AppGroup.isAvailable, config.hasCompletedOnboarding else { return }
if flowManager.isActive {
flowManager.extendSession()
} else {
flowManager.autoStartIfNeeded()
}
}
}
}
}
}
+79
View File
@@ -0,0 +1,79 @@
// AppPermissions.swift
// OSGKeyboard · Main App
//
// Central permission status for onboarding and Flow session startup.
import AVFoundation
import Speech
import UIKit
enum AppPermissions {
enum MicStatus: Equatable {
case undetermined
case granted
case denied
}
enum SpeechStatus: Equatable {
case undetermined
case granted
case denied
case restricted
}
static var micStatus: MicStatus {
switch AVAudioApplication.shared.recordPermission {
case .granted: return .granted
case .denied: return .denied
case .undetermined: return .undetermined
@unknown default: return .denied
}
}
static var speechStatus: SpeechStatus {
switch SFSpeechRecognizer.authorizationStatus() {
case .authorized: return .granted
case .denied: return .denied
case .restricted: return .restricted
case .notDetermined: return .undetermined
@unknown default: return .denied
}
}
/// Both permissions required for Flow voice sessions.
static var flowRequirementsMet: Bool {
micStatus == .granted && speechStatus == .granted
}
/// Show guided permission pages when any Flow permission is not granted.
static var needsPermissionGuidance: Bool {
micStatus != .granted || speechStatus != .granted
}
static func requestMicrophone() async -> Bool {
switch micStatus {
case .granted: return true
case .denied: return false
case .undetermined: return await AVAudioApplication.requestRecordPermission()
}
}
static func requestSpeechRecognition() async -> Bool {
switch speechStatus {
case .granted: return true
case .denied, .restricted: return false
case .undetermined:
return await withCheckedContinuation { (cont: CheckedContinuation<Bool, Never>) in
SFSpeechRecognizer.requestAuthorization { status in
cont.resume(returning: status == .authorized)
}
}
}
}
static func openSystemSettings() {
guard let url = URL(string: UIApplication.openSettingsURLString) else { return }
UIApplication.shared.open(url)
}
}
@@ -0,0 +1,396 @@
// FlowSessionManager.swift
// OSGKeyboard · Main App
//
// Session Owner for TypeWhisper-style Flow dictation: continuous
// `.playAndRecord` capture for the whole session, utterance gating for
// ASR, optional LLM polish, and App Group result delivery.
import Foundation
import AVFoundation
import Speech
import OSGKeyboardShared
@MainActor
final class FlowSessionManager: ObservableObject {
@Published private(set) var isActive = false
@Published private(set) var isStarting = false
@Published private(set) var sessionExpiresAt: Date?
/// Non-nil when continuous capture failed or permissions are missing.
@Published private(set) var sessionWarning: String?
private let capture = FlowContinuousCapture()
private let asr: ASRService = ASRServiceFactory.make()
private let polisher = PolishingService()
private let store = AppGroupStore()
private var pollingTask: Task<Void, Never>?
private var heartbeatTask: Task<Void, Never>?
private var expiryTask: Task<Void, Never>?
private var levelTask: Task<Void, Never>?
private var startTask: Task<Void, Never>?
private var isUtteranceRecording = false
private var finalizeTask: Task<Void, Never>?
private var asrTask: Task<Void, Never>?
private var currentPartial = ""
private var lastFinal = ""
init() {
Task { @MainActor [weak self] in
await self?.bootstrapFromStorageIfNeeded()
}
}
// MARK: - Public
/// Starts a Flow session: permissions continuous capture App Group active.
func startSession(duration: TimeInterval = FlowSessionKeys.defaultSessionDuration) {
guard AppGroup.isAvailable else {
debug("cannot start flow session: App Group unavailable")
return
}
if isActive {
extendSession(duration: duration)
return
}
startTask?.cancel()
startTask = Task { @MainActor [weak self] in
await self?.startSessionAsync(duration: duration)
}
}
/// Called on launch / foreground when onboarding is complete.
func autoStartIfNeeded() {
guard AppGroup.isAvailable else { return }
guard !isActive, !isStarting else { return }
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
return
}
let markedActive = AppGroup.defaults.bool(forKey: FlowSessionKeys.flowSessionActive)
if markedActive, FlowSessionBridge.remainingSessionDuration() != nil {
Task { await bootstrapFromStorageIfNeeded() }
return
}
startSession()
}
/// Reattach capture when the host app was killed but the session has not expired.
func bootstrapFromStorageIfNeeded() async {
guard AppGroup.isAvailable, !isActive else { return }
let markedActive = AppGroup.defaults.bool(forKey: FlowSessionKeys.flowSessionActive)
guard markedActive, let remaining = FlowSessionBridge.remainingSessionDuration(), remaining > 0 else {
if markedActive {
FlowSessionBridge.markSessionInactive()
}
return
}
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
return
}
isStarting = true
sessionWarning = nil
defer { isStarting = false }
do {
try capture.start()
} catch {
let message = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
sessionWarning = message
FlowSessionBridge.markSessionInactive()
debug("bootstrap capture failed: \(message)")
return
}
FlowSessionBridge.writeHeartbeat()
isActive = true
if let expires = FlowSessionBridge.sessionExpiresAt() {
sessionExpiresAt = Date(timeIntervalSince1970: expires)
}
startHeartbeat()
startPolling()
startLevelPublishing()
scheduleExpiry(after: remaining)
debug("Flow session restored (\(Int(remaining))s remaining)")
}
func endSession() {
guard isActive else { return }
debug("Flow session ended")
startTask?.cancel()
startTask = nil
pollingTask?.cancel()
pollingTask = nil
heartbeatTask?.cancel()
heartbeatTask = nil
expiryTask?.cancel()
expiryTask = nil
levelTask?.cancel()
levelTask = nil
finalizeTask?.cancel()
finalizeTask = nil
asrTask?.cancel()
asrTask = nil
if isUtteranceRecording {
capture.cancelUtterance()
asr.cancel()
isUtteranceRecording = false
}
capture.stop()
FlowSessionBridge.markSessionInactive()
isActive = false
sessionExpiresAt = nil
sessionWarning = nil
currentPartial = ""
lastFinal = ""
}
func extendSession(duration: TimeInterval = FlowSessionKeys.defaultSessionDuration) {
FlowSessionBridge.extendSession(by: duration)
sessionExpiresAt = Date().addingTimeInterval(duration)
scheduleExpiry(after: duration)
}
// MARK: - Session start
private func startSessionAsync(duration: TimeInterval) async {
isStarting = true
sessionWarning = nil
defer { isStarting = false }
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
return
}
do {
try capture.start()
} catch {
let message = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
sessionWarning = message
debug("continuous capture failed: \(message)")
return
}
FlowSessionBridge.markSessionActive(duration: duration)
isActive = true
sessionExpiresAt = Date().addingTimeInterval(duration)
startHeartbeat()
startPolling()
startLevelPublishing()
scheduleExpiry(after: duration)
debug("Flow session started (\(Int(duration))s), continuous capture running")
}
private func permissionWarningMessage() -> String {
if AppPermissions.micStatus != .granted {
return NSLocalizedString("flow.error.micRequired", comment: "")
}
return NSLocalizedString("flow.error.speechRequired", comment: "")
}
// MARK: - Polling
private func startPolling() {
pollingTask?.cancel()
pollingTask = Task { @MainActor [weak self] in
while !Task.isCancelled {
self?.handleKeyboardSignal()
try? await Task.sleep(nanoseconds: 100_000_000)
}
}
}
private func handleKeyboardSignal() {
switch FlowSessionBridge.recordingState() {
case .recording:
guard !isUtteranceRecording else { return }
beginUtterance()
case .stopped:
guard isUtteranceRecording else { return }
endUtterance()
case .aborted:
abortUtterance()
case .idle, .processing:
break
}
}
private func beginUtterance() {
guard capture.running else {
failUtterance(message: NSLocalizedString("flow.error.audioUnavailable", comment: ""))
return
}
finalizeTask?.cancel()
asrTask?.cancel()
asr.cancel()
currentPartial = ""
lastFinal = ""
let localeId = store.localeId
FlowSessionBridge.setTranscriptionLanguage(localeId)
FlowSessionBridge.clearPendingTranscription()
let locale = SpeechLocaleResolver.resolve(localeId)
let stream = capture.beginUtterance()
let events = asr.transcribe(stream: stream, locale: locale)
isUtteranceRecording = true
asrTask = Task { @MainActor [weak self] in
guard let self else { return }
for await event in events {
switch event {
case .capability:
break
case .partial(let text):
self.currentPartial = text
case .final(let text):
self.lastFinal = text.trimmingCharacters(in: .whitespacesAndNewlines)
self.currentPartial = ""
case .error(let message):
self.debug("asr error: \(message)")
if self.isUtteranceRecording {
self.failUtterance(message: message)
}
}
}
}
debug("utterance recording started")
}
private func endUtterance() {
isUtteranceRecording = false
capture.endUtterance()
FlowSessionBridge.setRecordingState(.processing)
finalizeTask?.cancel()
finalizeTask = Task { @MainActor [weak self] in
await self?.finalizeUtterance()
}
debug("utterance stopped, finalizing")
}
private func abortUtterance() {
isUtteranceRecording = false
finalizeTask?.cancel()
asrTask?.cancel()
asr.cancel()
capture.cancelUtterance()
currentPartial = ""
lastFinal = ""
FlowSessionBridge.setRecordingState(.idle)
debug("utterance aborted")
}
private func failUtterance(message: String) {
isUtteranceRecording = false
asrTask?.cancel()
asr.cancel()
capture.cancelUtterance()
currentPartial = ""
lastFinal = ""
FlowSessionBridge.storeTranscriptionError(message)
debug("utterance failed: \(message)")
}
private func finalizeUtterance() async {
let deadline = Date().addingTimeInterval(30)
while Date() < deadline {
if !lastFinal.isEmpty { break }
try? await Task.sleep(nanoseconds: 100_000_000)
}
var text = lastFinal.trimmingCharacters(in: .whitespacesAndNewlines)
if text.isEmpty {
text = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
}
guard !text.isEmpty else {
FlowSessionBridge.storeTranscriptionError(
NSLocalizedString("flow.error.noSpeech", comment: "")
)
return
}
let engineMode = store.engineMode
let modeId = store.modeId
let shouldPolish = engineMode != "local" && modeId == "polish"
if shouldPolish {
do {
let polished = try await polisher.polish(text)
FlowSessionBridge.storeTranscriptionResult(polished)
} catch {
FlowSessionBridge.storeTranscriptionResult(text)
}
} else {
FlowSessionBridge.storeTranscriptionResult(text)
}
currentPartial = ""
lastFinal = ""
debug("utterance finalized length=\(text.count)")
}
// MARK: - Level publishing (main thread only)
private func startLevelPublishing() {
levelTask?.cancel()
levelTask = Task { @MainActor [weak self] in
while !Task.isCancelled {
guard let self, self.isActive else { break }
let levels = self.capture.currentAudioLevels()
if levels.contains(where: { $0 > 0 }) {
FlowSessionBridge.storeAudioLevels(levels)
}
try? await Task.sleep(nanoseconds: 100_000_000)
}
}
}
// MARK: - Timers
private func startHeartbeat() {
heartbeatTask?.cancel()
FlowSessionBridge.writeHeartbeat()
heartbeatTask = Task { @MainActor [weak self] in
while !Task.isCancelled {
FlowSessionBridge.writeHeartbeat()
try? await Task.sleep(nanoseconds: 1_000_000_000)
guard self?.isActive == true else { break }
}
}
}
private func scheduleExpiry(after duration: TimeInterval) {
expiryTask?.cancel()
expiryTask = Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: UInt64(duration * 1_000_000_000))
guard !Task.isCancelled else { return }
self?.endSession()
}
}
private func debug(_ message: String) {
#if DEBUG
print("🌊[FlowSession] \(message)")
#endif
}
}
+11
View File
@@ -0,0 +1,11 @@
// LegalLinks.swift
// OSGKeyboard · Main App
import Foundation
enum LegalLinks {
/// Public privacy policy (GitHub Pages).
static var privacyPolicyURL: URL? {
URL(string: "https://hkgood.github.io/OSGKeyboard/privacy/")
}
}
@@ -0,0 +1,15 @@
// PermissionPrimer.swift
// OSGKeyboard · Main App
//
// Legacy entry point permission prompts now run inside Onboarding.
// Kept as a no-op so older call sites compile without re-firing TCC
// dialogs on every launch.
import Foundation
@MainActor
enum PermissionPrimer {
static func primeIfNeeded() async {
// Permissions are requested step-by-step in OnboardingView.
}
}
@@ -0,0 +1,146 @@
// DictationCaptureView.swift
// OSGKeyboard · Main App
//
// Host-app recording surface for keyboard handoff.
// Uses the shared `LiveDictationController` the same entry point as
// the keyboard preview sheet.
import SwiftUI
import OSGKeyboardShared
@MainActor
final class DictationSessionCoordinator: ObservableObject {
@Published var isPresenting: Bool = false
func present() {
isPresenting = true
}
func dismiss() {
isPresenting = false
}
}
struct DictationCaptureView: View {
@Environment(\.dismiss) private var dismiss
@Environment(\.themePalette) private var palette: ThemePalette
@ObservedObject var config: ProviderConfig
@ObservedObject var coordinator: DictationSessionCoordinator
@StateObject private var dictation = LiveDictationController()
@State private var statusText: String = "准备录音..."
@State private var isSaving: Bool = false
var body: some View {
ZStack {
palette.background.ignoresSafeArea()
VStack(spacing: Spacing.lg) {
Spacer()
Image(systemName: "mic.circle.fill")
.font(.system(size: 84, weight: .light))
.foregroundStyle(palette.accent)
Text(titleText)
.font(TypeStyle.title2)
.foregroundStyle(palette.textPrimary)
Text(statusText)
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg)
ProgressView(value: dictation.level, total: 1.0)
.tint(palette.accent)
.padding(.horizontal, Spacing.lg)
Spacer()
HStack(spacing: Spacing.sm) {
Button {
cancelAndClose()
} label: {
Text("取消")
.secondaryButton()
}
.buttonStyle(.plain)
.disabled(isSaving)
Button {
stopAndFinalize()
} label: {
Text("完成")
.primaryButton()
}
.buttonStyle(.plain)
.disabled(isSaving || dictation.phase != .recording)
}
.padding(.horizontal, Spacing.md)
.padding(.bottom, Spacing.lg)
}
}
.onAppear {
DictationBridge.setStatus(.requested)
startRecording()
}
.onDisappear {
dictation.stop()
}
.onChange(of: dictation.phase) { _, new in
switch new {
case .recording:
DictationBridge.setStatus(.recording)
statusText = config.isLocalEngine ? "正在实时识别..." : "正在听..."
case .processing:
DictationBridge.setStatus(.transcribing)
statusText = "处理中..."
case .requestingPermission:
statusText = "请求权限中..."
case .denied(let message):
DictationBridge.setStatus(.error, message: message)
statusText = message
case .error(let message):
DictationBridge.setStatus(.error, message: message)
statusText = message
case .idle:
break
}
}
.onChange(of: dictation.currentPartial) { _, new in
guard config.isLocalEngine, !new.isEmpty else { return }
statusText = new
}
.onChange(of: dictation.lastFinal) { _, new in
guard !new.isEmpty else { return }
saveAndClose(new)
}
}
private var titleText: String {
isSaving ? "保存中..." : "语音输入"
}
private func startRecording() {
Task { await dictation.start(localeId: config.localeId) }
}
private func stopAndFinalize() {
dictation.stop()
statusText = "等待识别结果..."
}
private func cancelAndClose() {
dictation.stop()
DictationBridge.setStatus(.cancelled)
coordinator.dismiss()
dismiss()
}
private func saveAndClose(_ transcript: String) {
guard !isSaving else { return }
isSaving = true
DictationBridge.storePendingTranscript(transcript)
coordinator.dismiss()
dismiss()
}
}
+75
View File
@@ -12,6 +12,7 @@ struct HomeView: View {
@Environment(\.themePalette) private var palette: ThemePalette
@ObservedObject var config = ProviderConfig.shared
@EnvironmentObject private var flowManager: FlowSessionManager
@State private var showSettings = false
@State private var showKeyboardPreview = false
@@ -21,6 +22,9 @@ struct HomeView: View {
VStack(spacing: 0) {
statusHeader
.padding(.top, Spacing.xl)
flowSessionCard
.padding(.horizontal, Spacing.md)
.padding(.top, Spacing.md)
Spacer()
heroButton
Spacer()
@@ -37,6 +41,77 @@ struct HomeView: View {
}
}
// MARK: - Flow session
private var flowSessionCard: some View {
VStack(alignment: .leading, spacing: Spacing.xs) {
HStack {
Circle()
.fill(flowStatusColor)
.frame(width: 8, height: 8)
Text(flowStatusTitle)
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
Spacer()
if flowManager.isActive, let expires = flowManager.sessionExpiresAt {
Text(expires, style: .timer)
.font(TypeStyle.caption2)
.foregroundStyle(palette.textTertiary)
.monospacedDigit()
}
}
Text("home.flow.hint")
.font(TypeStyle.caption2)
.foregroundStyle(palette.textTertiary)
.fixedSize(horizontal: false, vertical: true)
if let warning = flowManager.sessionWarning {
Text(warning)
.font(TypeStyle.caption2)
.foregroundStyle(palette.warning)
.fixedSize(horizontal: false, vertical: true)
if !AppPermissions.flowRequirementsMet {
Button {
AppPermissions.openSystemSettings()
} label: {
Text("home.flow.openSettings")
.font(TypeStyle.caption)
.foregroundStyle(palette.accent)
}
.buttonStyle(.plain)
}
}
if flowManager.isActive {
Button {
flowManager.endSession()
} label: {
Text("home.flow.end")
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
}
.buttonStyle(.plain)
}
}
.padding(Spacing.sm)
.background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.large, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: Radius.large, style: .continuous)
.stroke(palette.divider, lineWidth: 0.5)
)
}
private var flowStatusColor: Color {
if flowManager.isActive { return palette.success }
if flowManager.isStarting { return palette.accent }
if flowManager.sessionWarning != nil { return palette.warning }
return palette.warning
}
private var flowStatusTitle: LocalizedStringKey {
if flowManager.isActive { return "home.flow.active" }
if flowManager.isStarting { return "home.flow.starting" }
return "home.flow.inactive"
}
// MARK: - Header
private var statusHeader: some View {
+120 -158
View File
@@ -1,82 +1,32 @@
// KeyboardPreviewSheet.swift
// OSGKeyboard · Main App (Debug)
//
// In-app preview of the keyboard extension. Renders a stand-in
// `KeyboardPreviewStub` so the user can see what the real extension
// looks like, AND drives a *real* `PreviewASRController` so tapping
// the disc actually records from the mic and runs the shared ASR
// service (`SpeechAnalyzer` on iOS 26+),
// and lands recognized text in the top textbox. Without the real ASR
// the preview was a static mock "the text never appears" was a
// fair review note.
// In-app preview of the keyboard extension. Drives the shared
// `LiveDictationController` so tapping record exercises the same
// on-device ASR pipeline as host-app dictation handoff.
//
// Lifecycle (local engine = "transcribe" only, no LLM):
// tap start ASR (idempotent re-entry guard)
// ASR emits .partial / .final
// currentPartial updates the transcript line in real time
// tap stop ASR
// lastFinal event lands
// onChange in this view appends to typedText
// controller resets
// Local engine: partial transcripts stream into the text box live.
// Cloud engine: final transcript is appended when recording stops.
import SwiftUI
import OSGKeyboardShared
struct KeyboardPreviewSheet: View {
@Environment(\.themePalette) private var palette: ThemePalette
@Environment(\.dismiss) private var dismiss
@ObservedObject private var config = ProviderConfig.shared
@StateObject private var asr = PreviewASRController()
@StateObject private var dictation = LiveDictationController()
@State private var showSettings = false
/// Accumulates text in the top textbox. The real keyboard extension
/// inserts directly via `textDocumentProxy`; this preview mirrors
/// that in a parallel `@State` so the user can verify the flow.
@State private var typedText: String = ""
/// Cached snapshot of the previous lastFinal so the .onChange
/// doesn't fire on every body re-render only on real changes.
@State private var lastFinalSeen: String = ""
private enum StubPhase { case idle, recording, processing }
/// Visible phase for the stub. Driven by the real ASR controller
/// when the controller is `recording` we show the recording
/// state; otherwise we show `processing` while a final is in
/// flight and `idle` otherwise.
private var stubPhase: KeyboardPreviewStub.Phase {
switch asr.phase {
case .recording: return .recording
case .processing: return .processing
case .idle: return .idle
case .requestingPermission:
return .recording
case .denied, .error:
return .idle
}
}
/// The transcript line the stub shows under the chips. While
/// recording we surface the live ASR partial; otherwise we surface
/// any error or stay quiet.
private var stubTranscript: String {
switch asr.phase {
case .recording:
return asr.currentPartial.isEmpty ? " " : asr.currentPartial
case .error(let m):
return m
case .denied(let m):
return m
default:
return ""
}
}
/// Text in the box when the current dictation session started.
@State private var dictationAnchorText: String = ""
var body: some View {
ZStack {
palette.background.ignoresSafeArea()
VStack(spacing: 0) {
VStack(spacing: Spacing.md) {
VStack(spacing: Spacing.md) {
Text("preview.title")
.font(TypeStyle.title2)
@@ -87,55 +37,62 @@ struct KeyboardPreviewSheet: View {
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg)
mockTextField.padding(.horizontal, Spacing.md)
controls
.padding(.horizontal, Spacing.md)
}
.padding(.top, Spacing.lg)
Spacer(minLength: 0)
keyboardBlock
Spacer()
}
}
.sheet(isPresented: $showSettings) {
SettingsView()
}
.onChange(of: asr.lastFinal) { _, new in
.onChange(of: dictation.currentPartial) { _, partial in
guard config.isLocalEngine, isDictationActive else { return }
applyLiveTranscript(partial)
}
.onChange(of: dictation.lastFinal) { _, new in
guard !new.isEmpty, new != lastFinalSeen else { return }
lastFinalSeen = new
insertRecognizedText(new)
asr.reset()
if config.isLocalEngine {
applyLiveTranscript(new)
dictationAnchorText = typedText
} else {
insertRecognizedText(new)
}
dictation.reset()
}
// Tear down the ASR pipeline when the sheet leaves the screen
// (preview dismissed, app backgrounded mid-recording, etc.).
// Without this, a leftover `asrTask` keeps the AVAudioSession
// active and the mic permission in use after the user has
// moved on. `asr.stop()` is idempotent it no-ops on
// `.idle`/`.denied`/`.error` so it's safe to call here
// even when the disc is not currently recording.
.onDisappear {
asr.stop()
dictation.stop()
}
}
/// Real `TextField` (was a static placeholder HStack before the
/// first fix). The user can both type into it AND see recognized
/// text land in it as the real ASR fires `.final`.
private var mockTextField: some View {
HStack(spacing: Spacing.xs) {
Image(systemName: "text.cursor")
.foregroundStyle(palette.textSecondary)
TextField(LocalizedStringKey("preview.placeholder"), text: $typedText, axis: .vertical)
.lineLimit(1...4)
.textFieldStyle(.plain)
VStack(alignment: .leading, spacing: Spacing.xs) {
HStack {
Image(systemName: "text.cursor")
.foregroundStyle(palette.textSecondary)
Text(statusTitle)
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
Spacer()
if !typedText.isEmpty {
Button {
typedText = ""
} label: {
Image(systemName: "xmark.circle.fill")
.foregroundStyle(palette.textTertiary)
}
.buttonStyle(.plain)
.accessibilityLabel("preview.clear")
}
}
TextEditor(text: $typedText)
.frame(minHeight: 260)
.scrollContentBackground(.hidden)
.foregroundStyle(palette.textPrimary)
.tint(palette.accent)
if !typedText.isEmpty {
Button {
typedText = ""
} label: {
Image(systemName: "xmark.circle.fill")
.foregroundStyle(palette.textTertiary)
}
.buttonStyle(.plain)
.accessibilityLabel("preview.clear")
}
.padding(.horizontal, 2)
}
.padding(Spacing.md)
.background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.medium))
@@ -145,94 +102,99 @@ struct KeyboardPreviewSheet: View {
)
}
private var keyboardBlock: some View {
VStack(spacing: 0) {
Rectangle()
.fill(palette.divider)
.frame(height: 0.5)
KeyboardPreviewStub(
phase: stubPhase,
level: asr.level,
transcript: stubTranscript,
modeId: config.modeId,
localeId: config.localeId,
onTap: cyclePhase,
openSettings: { showSettings = true },
onModeCycle: cycleMode,
onLocaleCycle: cycleLocale
)
private var controls: some View {
VStack(spacing: Spacing.xs) {
HStack(spacing: Spacing.sm) {
Button {
toggleRecording()
} label: {
Label(isRecording ? "停止录音" : "开始录音", systemImage: isRecording ? "stop.circle.fill" : "mic.circle.fill")
.primaryButton()
}
.buttonStyle(.plain)
Button {
showSettings = true
} label: {
Label("设置", systemImage: "gearshape")
.secondaryButton()
}
.buttonStyle(.plain)
}
Text(serviceLabel)
.font(TypeStyle.caption2)
.foregroundStyle(palette.textTertiary)
.frame(maxWidth: .infinity, alignment: .center)
.lineLimit(2)
.multilineTextAlignment(.center)
}
}
/// Tap on the disc. Drives the *real* ASR pipeline the previous
/// mock that just toggled a hardcoded phase is gone.
private func cyclePhase() {
private var serviceLabel: String {
EngineServiceLabel.summary(
engineMode: config.engineMode,
providerId: config.providerId,
model: config.model
)
}
private var isRecording: Bool {
if case .recording = dictation.phase { return true }
return false
}
private var isDictationActive: Bool {
switch dictation.phase {
case .recording, .processing: return true
default: return false
}
}
private var statusTitle: String {
switch dictation.phase {
case .idle: return "输入框"
case .requestingPermission: return "请求权限中..."
case .recording: return config.isLocalEngine ? "正在实时识别..." : "正在录音..."
case .processing: return "识别中..."
case .denied(let message), .error(let message):
return message
}
}
private func toggleRecording() {
withAnimation(Motion.quick) {
switch asr.phase {
switch dictation.phase {
case .idle, .denied, .error:
let locale = resolveLocale(config.localeId)
Task { await asr.start(locale: locale) }
startRecording()
case .recording:
asr.stop()
stopAndFinalize()
case .requestingPermission, .processing:
break
}
}
}
/// Cycle the input mode on tap of the mode chip. The order mirrors
/// the Settings picker (`off` `transcribe` `polish` wrap)
/// so the user sees the same surface in both places.
///
/// Note: when the user picks `off` we *also* stop any in-flight
/// recording leaving the disc mid-recording in an "off" mode
/// would be a confusing state (the user is recording but the
/// keyboard says it won't insert anything).
private func cycleMode() {
let order = ["off", "transcribe", "polish"]
let current = order.firstIndex(of: config.modeId) ?? 0
let next = order[(current + 1) % order.count]
config.modeId = next
if next == "off" && asr.phase == .recording {
asr.stop()
}
private func startRecording() {
dictationAnchorText = typedText
lastFinalSeen = ""
Task { await dictation.start(localeId: config.localeId) }
}
/// Cycle the locale on tap of the locale chip. Same order as
/// `staticLocales` in `SettingsView.swift` so both surfaces stay
/// in sync. When the user picks a new locale we *also* stop any
/// in-flight recording ASR sessions are bound to the locale
/// they were started with, and continuing to feed buffers into a
/// stale session would produce garbage in the next `.final`.
private func cycleLocale() {
let order = ["auto", "zh-Hans", "zh-Hant", "en-US", "ja-JP", "ko-KR"]
let current = order.firstIndex(of: config.localeId) ?? 0
let next = order[(current + 1) % order.count]
config.localeId = next
if asr.phase == .recording {
asr.stop()
}
private func stopAndFinalize() {
dictation.stop()
}
private func resolveLocale(_ id: String) -> Locale {
if id == "auto" { return .current }
return Locale(identifier: id)
private func applyLiveTranscript(_ transcript: String) {
let composed = DictationTextComposer.compose(anchor: dictationAnchorText, live: transcript)
guard composed != typedText else { return }
typedText = composed
}
/// Append the recognized transcript to the textbox, with a leading
/// space when the existing text doesn't already end in whitespace.
/// Matches what `textDocumentProxy.insertText` does for the real
/// keyboard when the user's draft has no trailing whitespace.
private func insertRecognizedText(_ recognized: String) {
let trimmed = recognized.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
if typedText.isEmpty {
typedText = trimmed
} else if typedText.last == " " || typedText.last == "\n" {
typedText += trimmed
} else {
typedText += " " + trimmed
}
typedText = DictationTextComposer.compose(anchor: typedText, live: trimmed)
}
}
+240 -60
View File
@@ -1,32 +1,26 @@
// OnboardingView.swift
// OSGKeyboard · Main App
//
// Three-step onboarding:
//
// 1) Welcome what the app does, in one sentence
// 2) Enable Settings General Keyboards Add Allow Full Access
// 3) Setup pick a provider, paste a key
//
// We deliberately do NOT use `TabView` with `.page` style for the
// pager. That style wraps the content in a `UIPageViewController`,
// and `UIPageViewController` has a long-standing behavior on some iOS
// versions where
// the keyboard-showing layout reflow on a `TextField` focus is
// misread as a horizontal swipe the page jumps back to step 1
// the moment the user starts typing. Replacing the TabView with a
// `ZStack`-based conditional view sidesteps the bug entirely; we
// give up the swipe-to-page gesture, but the Back/Next buttons at
// the bottom (and the page dots) are the canonical onboarding
// affordance and the user is never more than one tap from the next
// page anyway.
//
// Visual style: one large accent surface, generous whitespace, single CTA
// at the bottom. No tipsy animations, no cheerful illustrations every
// pixel is doing one job.
// Five-step onboarding:
// 1) Welcome
// 2) Microphone permission
// 3) Speech recognition permission
// 4) Enable keyboard + Allow Full Access
// 5) Engine / API setup
import SwiftUI
import OSGKeyboardShared
private enum OnboardingPage: Int, CaseIterable {
case welcome = 0
case microphone
case speech
case keyboard
case api
static let count = 5
}
struct OnboardingView: View {
@Environment(\.themePalette) private var palette: ThemePalette
@@ -38,10 +32,12 @@ struct OnboardingView: View {
palette.background.ignoresSafeArea()
VStack(spacing: 0) {
Group {
switch page {
case 0: WelcomePage()
case 1: EnableKeyboardPage()
default: APISetupPage(config: config)
switch OnboardingPage(rawValue: page) ?? .welcome {
case .welcome: WelcomePage()
case .microphone: MicPermissionPage()
case .speech: SpeechPermissionPage()
case .keyboard: EnableKeyboardPage()
case .api: APISetupPage(config: config)
}
}
.frame(maxWidth: .infinity, maxHeight: .infinity)
@@ -59,7 +55,7 @@ struct OnboardingView: View {
private var pageDots: some View {
HStack(spacing: 6) {
ForEach(0..<3, id: \.self) { i in
ForEach(0..<OnboardingPage.count, id: \.self) { i in
Capsule()
.fill(i == page ? palette.accent : Color.white.opacity(0.18))
.frame(width: i == page ? 18 : 6, height: 6)
@@ -68,6 +64,19 @@ struct OnboardingView: View {
}
}
private var isLastPage: Bool { page == OnboardingPage.api.rawValue }
private var canAdvance: Bool {
switch OnboardingPage(rawValue: page) ?? .welcome {
case .welcome, .keyboard, .api:
return page != OnboardingPage.api.rawValue || config.isConfigured
case .microphone:
return AppPermissions.micStatus != .undetermined
case .speech:
return AppPermissions.speechStatus != .undetermined
}
}
@ViewBuilder
private var bottomBar: some View {
HStack(spacing: Spacing.sm) {
@@ -88,31 +97,31 @@ struct OnboardingView: View {
Button {
withAnimation(Motion.soft) {
if page < 2 { page += 1 }
if isLastPage {
config.hasCompletedOnboarding = true
} else {
page += 1
}
}
} label: {
Text(page == 2
? (config.isConfigured
? NSLocalizedString("common.done", comment: "")
: NSLocalizedString("common.continue", comment: ""))
Text(isLastPage
? NSLocalizedString("common.done", comment: "")
: NSLocalizedString("common.next", comment: ""))
.font(TypeStyle.headline)
.frame(maxWidth: .infinity, minHeight: 50)
.background(
(page == 2 && !config.isConfigured) ? palette.surfaceElevated : palette.accent,
canAdvance ? palette.accent : palette.surfaceElevated,
in: RoundedRectangle(cornerRadius: Radius.large, style: .continuous)
)
.foregroundStyle(
(page == 2 && !config.isConfigured) ? palette.textSecondary : palette.textOnAccent
)
.foregroundStyle(canAdvance ? palette.textOnAccent : palette.textSecondary)
}
.buttonStyle(.plain)
.disabled(page == 2 && !config.isConfigured)
.disabled(!canAdvance)
}
}
}
// MARK: - Page 1: Welcome
// MARK: - Welcome
private struct WelcomePage: View {
@Environment(\.themePalette) private var palette: ThemePalette
@@ -141,6 +150,14 @@ private struct WelcomePage: View {
.padding(.horizontal, Spacing.xl)
PrivacyFootnote()
.padding(.top, Spacing.lg)
if let url = LegalLinks.privacyPolicyURL {
Link(destination: url) {
Text("legal.privacyPolicy")
.font(TypeStyle.caption)
.foregroundStyle(palette.accent)
}
.padding(.top, Spacing.xs)
}
Spacer()
}
}
@@ -151,15 +168,9 @@ private struct PrivacyFootnote: View {
var body: some View {
VStack(alignment: .leading, spacing: 8) {
footnoteRow(icon: "lock.fill",
title: "privacy.audio.title",
body: "privacy.audio.body")
footnoteRow(icon: "wifi",
title: "privacy.network.title",
body: "privacy.network.body")
footnoteRow(icon: "keyboard",
title: "privacy.universal.title",
body: "privacy.universal.body")
footnoteRow(icon: "lock.fill", title: "privacy.audio.title", body: "privacy.audio.body")
footnoteRow(icon: "wifi", title: "privacy.network.title", body: "privacy.network.body")
footnoteRow(icon: "keyboard", title: "privacy.universal.title", body: "privacy.universal.body")
}
.padding(Spacing.md)
.background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.large, style: .continuous))
@@ -170,9 +181,6 @@ private struct PrivacyFootnote: View {
.padding(.horizontal, Spacing.md)
}
// `LocalizedStringKey` (not `String`) so the call-site string
// literals are auto-looked-up in Localizable.strings. Passing a
// plain `String` would just print the key.
private func footnoteRow(icon: String, title: LocalizedStringKey, body: LocalizedStringKey) -> some View {
HStack(alignment: .top, spacing: Spacing.xs) {
Image(systemName: icon)
@@ -192,7 +200,180 @@ private struct PrivacyFootnote: View {
}
}
// MARK: - Page 2: Enable keyboard
// MARK: - Permission pages
private struct MicPermissionPage: View {
@Environment(\.themePalette) private var palette: ThemePalette
@State private var status = AppPermissions.micStatus
@State private var isRequesting = false
var body: some View {
PermissionPageLayout(
icon: "mic.fill",
title: "onboarding.permission.mic.title",
detail: "onboarding.permission.mic.body",
status: statusLabel,
statusColor: statusColor,
primaryTitle: primaryButtonTitle,
primaryDisabled: isRequesting || status == .granted,
onPrimary: { Task { await request() } },
secondaryTitle: status == .denied ? "onboarding.permission.openSettings" : nil,
onSecondary: status == .denied ? { AppPermissions.openSystemSettings() } : nil,
deniedHint: status == .denied ? "onboarding.permission.mic.deniedHint" : nil
)
.onAppear { status = AppPermissions.micStatus }
}
private var statusLabel: LocalizedStringKey {
switch status {
case .undetermined: return "onboarding.permission.status.undetermined"
case .granted: return "onboarding.permission.status.granted"
case .denied: return "onboarding.permission.status.denied"
}
}
private var statusColor: Color {
switch status {
case .granted: return palette.success
case .denied: return palette.warning
case .undetermined: return palette.textTertiary
}
}
private var primaryButtonTitle: LocalizedStringKey {
status == .granted ? "onboarding.permission.status.granted" : "onboarding.permission.mic.allow"
}
private func request() async {
isRequesting = true
_ = await AppPermissions.requestMicrophone()
status = AppPermissions.micStatus
isRequesting = false
}
}
private struct SpeechPermissionPage: View {
@Environment(\.themePalette) private var palette: ThemePalette
@State private var isRequesting = false
var body: some View {
PermissionPageLayout(
icon: "waveform.badge.mic",
title: "onboarding.permission.speech.title",
detail: "onboarding.permission.speech.body",
status: statusLabel,
statusColor: statusColor,
primaryTitle: primaryButtonTitle,
primaryDisabled: isRequesting || AppPermissions.speechStatus == .granted,
onPrimary: { Task { await request() } },
secondaryTitle: speechDenied ? "onboarding.permission.openSettings" : nil,
onSecondary: speechDenied ? { AppPermissions.openSystemSettings() } : nil,
deniedHint: speechDenied ? "onboarding.permission.speech.deniedHint" : nil
)
}
private var speechDenied: Bool {
switch AppPermissions.speechStatus {
case .denied, .restricted: return true
default: return false
}
}
private var statusLabel: LocalizedStringKey {
switch AppPermissions.speechStatus {
case .undetermined: return "onboarding.permission.status.undetermined"
case .granted: return "onboarding.permission.status.granted"
case .denied, .restricted: return "onboarding.permission.status.denied"
}
}
private var statusColor: Color {
switch AppPermissions.speechStatus {
case .granted: return palette.success
case .denied, .restricted: return palette.warning
case .undetermined: return palette.textTertiary
}
}
private var primaryButtonTitle: LocalizedStringKey {
AppPermissions.speechStatus == .granted
? "onboarding.permission.status.granted"
: "onboarding.permission.speech.allow"
}
private func request() async {
isRequesting = true
_ = await AppPermissions.requestSpeechRecognition()
isRequesting = false
}
}
private struct PermissionPageLayout: View {
@Environment(\.themePalette) private var palette: ThemePalette
let icon: String
let title: LocalizedStringKey
let detail: LocalizedStringKey
let status: LocalizedStringKey
let statusColor: Color
let primaryTitle: LocalizedStringKey
let primaryDisabled: Bool
let onPrimary: () -> Void
var secondaryTitle: LocalizedStringKey? = nil
var onSecondary: (() -> Void)? = nil
var deniedHint: LocalizedStringKey? = nil
var body: some View {
VStack(spacing: Spacing.lg) {
Spacer()
Image(systemName: icon)
.font(.system(size: 64, weight: .light))
.foregroundStyle(palette.accent)
VStack(spacing: Spacing.sm) {
Text(title)
.font(TypeStyle.title2)
.foregroundStyle(palette.textPrimary)
Text(detail)
.font(TypeStyle.body)
.foregroundStyle(palette.textSecondary)
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg)
}
HStack(spacing: 6) {
Circle().fill(statusColor).frame(width: 8, height: 8)
Text(status)
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
}
if let deniedHint {
Text(deniedHint)
.font(TypeStyle.caption2)
.foregroundStyle(palette.warning)
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg)
}
VStack(spacing: Spacing.xs) {
Button(action: onPrimary) {
Text(primaryTitle)
.primaryButton()
}
.buttonStyle(.plain)
.disabled(primaryDisabled)
if let secondaryTitle, let onSecondary {
Button(action: onSecondary) {
Text(secondaryTitle)
.secondaryButton()
}
.buttonStyle(.plain)
}
}
.padding(.horizontal, Spacing.md)
Spacer()
}
}
}
// MARK: - Enable keyboard
private struct EnableKeyboardPage: View {
@Environment(\.themePalette) private var palette: ThemePalette
@@ -207,6 +388,11 @@ private struct EnableKeyboardPage: View {
Text("onboarding.enable.title")
.font(TypeStyle.title2)
.foregroundStyle(palette.textPrimary)
Text("onboarding.enable.fullAccessNote")
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg)
}
VStack(alignment: .leading, spacing: Spacing.sm) {
step(num: 1, text: NSLocalizedString("onboarding.enable.step1", comment: ""))
@@ -218,13 +404,12 @@ private struct EnableKeyboardPage: View {
.padding(.horizontal, Spacing.md)
Button {
if let url = URL(string: UIApplication.openSettingsURLString) {
UIApplication.shared.open(url)
}
AppPermissions.openSystemSettings()
} label: {
Label(LocalizedStringKey("onboarding.enable.openSettings"), systemImage: "arrow.up.right.square")
.primaryButton()
}
.buttonStyle(.plain)
.padding(.horizontal, Spacing.md)
Spacer()
}
@@ -245,7 +430,7 @@ private struct EnableKeyboardPage: View {
}
}
// MARK: - Page 3: API setup
// MARK: - API setup
private struct APISetupPage: View {
@Environment(\.themePalette) private var palette: ThemePalette
@@ -267,20 +452,15 @@ private struct APISetupPage: View {
.padding(.horizontal, Spacing.md)
.padding(.top, Spacing.lg)
// Same Engine picker as Settings see EnginePickerSection.
EnginePickerSection(config: config)
.padding(.horizontal, Spacing.md)
if config.engineMode == "cloud" {
ProviderPickerSection(config: config)
.padding(.horizontal, Spacing.md)
APISettingsCard(config: config)
.padding(.horizontal, Spacing.md)
} else {
// Local path: no LLM, no API key needed. Show a short
// confirmation so the user understands "no further
// setup required".
VStack(alignment: .leading, spacing: Spacing.xs) {
HStack(spacing: Spacing.sm) {
Image(systemName: "checkmark.seal.fill")
+4 -457
View File
@@ -1,462 +1,9 @@
// PreviewASRController.swift
// OSGKeyboard · Main App (Debug)
// OSGKeyboard · Main App
//
// Self-contained ASR controller for the in-app keyboard preview sheet.
// Owns its own AVAudioEngine + AVAudioSession, downsamples to 16 kHz
// mono Float32, and feeds the `AudioBufferSnapshot` stream to the
// shared `ASRService` (the same pipeline the real keyboard extension
// uses, so the preview exercises the *real* iOS speech APIs, not a
// stub). Without this the in-app preview was a hardcoded transcript
// and "did you actually call SFSpeechRecognizer?" was a fair review
// note.
//
// Why not reuse `AudioCaptureService` from the extension? It lives in
// `OSGKeyboardExt`, an `app-extension` target the main app can't
// import its symbols. We could move it to `OSGKeyboardShared`, but
// `AVAudioSession` lifecycle differs enough between a keyboard
// extension (no background, no recording entitlement surprise) and a
// foreground app that a copy here is the lesser evil.
// Legacy name kept for existing call sites and tests. The implementation
// lives in `OSGKeyboardShared` as `LiveDictationController`.
import Foundation
import AVFoundation
import Speech
import OSGKeyboardShared
@MainActor
final class PreviewASRController: ObservableObject {
enum Phase: Equatable {
case idle
case requestingPermission
case recording
case processing
case denied(String)
case error(String)
}
@Published private(set) var phase: Phase = .idle
/// Normalized 0...1 RMS for the disc level meter. Polled from the
/// audio tap via `Task { @MainActor in ... }` the tap itself
/// runs on a real-time audio thread, so we never touch published
/// state from there.
@Published private(set) var level: Double = 0
@Published private(set) var currentPartial: String = ""
@Published private(set) var errorMessage: String?
/// Set when a `.final` ASR event lands. The owning sheet observes
/// this and appends the text to its textbox, then clears it so the
/// next recording starts from zero.
@Published var lastFinal: String = ""
private let asr: ASRService = ASRServiceFactory.make()
private let audioEngine = AVAudioEngine()
/// `internal` (not `private`) so the regression test in
/// `OSGKeyboardTests/PreviewASRControllerStateTests.swift` can
/// install a known consumer task and assert `stop()` doesn't
/// cancel it. The class is `@MainActor`-isolated, so the
/// natural Swift 6 isolation rules still prevent production
/// code outside the class from racing on it.
var asrTask: Task<Void, Never>?
private var bufferContinuation: AsyncStream<AudioBufferSnapshot>.Continuation?
private var didConfigureAudioSession = false
private var didInstallTap = false
func start(locale: Locale) async {
// Re-entry guard: ignore taps that arrive while we're already
// running. (The sheet's `cyclePhase` is also guarded, but
// async race windows are easier to lock down here.)
switch phase {
case .recording, .requestingPermission, .processing:
return
default:
break
}
// Cancel any leftover consumer task from a previous recording.
// Normally `stop()` lets the task run to completion (so it can
// see the `.final` and transition out of `.processing`), but if
// the user smashed the disc twice stop, then immediately
// start the previous task might still be draining. Cancel it
// here so we don't have two consumer tasks fighting over the
// same `events` stream.
asrTask?.cancel()
asrTask = nil
phase = .requestingPermission
currentPartial = ""
lastFinal = ""
errorMessage = nil
level = 0
// 1. Microphone permission. The helper is `nonisolated` so the
// (iOS < 17) callback closure does not inherit `@MainActor`
// `AVAudioSession.requestRecordPermission` delivers on a TCC
// reply queue, and a `@MainActor`-inferred closure body there
// hits `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift`.
let micGranted = await Self.requestMicrophonePermission()
guard micGranted else {
phase = .denied(NSLocalizedString("keyboard.denied.mic", comment: ""))
return
}
// 2. Speech recognition permission. Same reasoning as above:
// the callback fires on TCC's reply queue, NOT the main queue.
let speechGranted = await Self.requestSpeechRecognitionPermission()
guard speechGranted else {
phase = .denied(NSLocalizedString("keyboard.denied.speech", comment: ""))
return
}
// 3. Audio session only configure once per process.
//
// Category is `.record` (not `.playAndRecord`) because the
// preview never plays back audio it just records from the
// mic and hands the buffers to `SpeechAnalyzer`. On the
// iOS Simulator, `.playAndRecord` requires the
// `AURemoteIO` Audio Unit's *output* side to also be
// enabled, but the simulator's "speaker" reports a 0 Hz
// hardware format, so `AURemoteIO::enable` fails with
// `kAudioUnitErr_FormatNotSupported` (-10851) and any
// subsequent `installTap` traps with "Failed to create tap
// due to format mismatch". `.record` skips the output
// side entirely, so the simulator can record.
//
// The real keyboard extension (`OSGKeyboardExt`) keeps
// `.playAndRecord` because it runs on a real device where
// the output side has a real hardware format, and may want
// to play click sounds / haptic feedback. Only the preview
// needs the simulator-friendly category.
if !didConfigureAudioSession {
do {
let session = AVAudioSession.sharedInstance()
try session.setCategory(.record,
mode: .measurement,
options: [])
try session.setActive(true, options: .notifyOthersOnDeactivation)
didConfigureAudioSession = true
} catch {
phase = .error(String.localizedStringWithFormat(
NSLocalizedString("preview.error.audioSession", comment: ""),
error.localizedDescription
))
return
}
}
// 4. Spin up the engine + ASR.
phase = .recording
startEngineAndASR(locale: locale)
}
func stop() {
// Don't `asrTask?.cancel()` here see the comment in
// `startEngineAndASR` for the full rationale. Short version:
// cancelling the consumer task at the same moment we close the
// audio stream also triggers the producer's
// `continuation.onTermination self?.cancel()` cascade, which
// marks the producer's outer task as cancelled and skips the
// `.final` event. The UI is then left in `.processing` forever
// because no one schedules the transition out. The consumer
// task naturally exits when `events` finishes, so the right
// thing is to let it run.
//
// If a previous `asrTask` is somehow still running (e.g. the
// user smashed the disc twice quickly), `start()` cancels it
// at the entry point as a safety net.
if didInstallTap {
audioEngine.inputNode.removeTap(onBus: 0)
didInstallTap = false
}
if audioEngine.isRunning {
audioEngine.stop()
}
bufferContinuation?.finish()
bufferContinuation = nil
if phase == .recording {
phase = .processing
}
// Deactivate so the user's music resumes if the preview is
// dismissed mid-recording.
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
// Safety net: if the ASR pipeline never produces a `.final`
// (analyzer hang, system glitch, dropped continuation), force
// the UI back to idle after a short delay so the user isn't
// stuck. Normal recordings complete well under a second, so
// the 3-second budget is only hit on the unhappy path; if the
// pipeline finishes first and flips the phase to `.idle` (or
// `.error`), the check below no-ops.
Task { @MainActor [weak self] in
try? await Task.sleep(for: .seconds(3))
guard let self else { return }
if self.phase == .processing {
self.phase = .idle
}
}
}
func reset() {
// Called by the sheet after appending `lastFinal` to the textbox,
// so the next recording can produce a fresh final without us
// double-appending.
lastFinal = ""
if phase == .processing {
phase = .idle
}
}
// MARK: - Engine + ASR
private func startEngineAndASR(locale: Locale) {
let inputNode = audioEngine.inputNode
let hwFormat = inputNode.outputFormat(forBus: 0)
// Pre-flight check: a placeholder / unconfigured input bus
// reports `sampleRate == 0` (or `channelCount == 0`).
// `installTap` on such a bus traps with "Failed to create
// tap due to format mismatch" (an NSException, not a Swift
// `Error`, so we can't `try`/`catch` it). The safest fix
// is to refuse the tap up front and surface a clear
// `.error` phase instead of crashing the app. We've seen
// this on the iOS Simulator when the host's microphone
// permission isn't granted to CoreSimulator, and on
// devices where the audio session is in an unexpected
// state from a previous foreground/background transition.
guard hwFormat.sampleRate > 0, hwFormat.channelCount > 0 else {
phase = .error(
String.localizedStringWithFormat(
NSLocalizedString("preview.error.micUnavailable", comment: ""),
hwFormat.sampleRate,
Int(hwFormat.channelCount)
)
)
return
}
let targetSampleRate: Double = 16_000
guard let targetFormat = AVAudioFormat(
commonFormat: .pcmFormatFloat32,
sampleRate: targetSampleRate,
channels: 1,
interleaved: false
) else {
phase = .error(NSLocalizedString("preview.error.formatCreate", comment: ""))
return
}
guard let converter = AVAudioConverter(from: hwFormat, to: targetFormat) else {
phase = .error(NSLocalizedString("preview.error.converterCreate", comment: ""))
return
}
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
self.bufferContinuation = continuation
// Tap the hardware input. The closure passed to `installTap` runs
// on the AVAudioEngine real-time audio thread. In Swift 6 strict
// concurrency, a closure literal defined inside a `@MainActor`
// method inherits `@MainActor` isolation, which would trip
// `dispatch_assert_queue_fail` on first invocation from the
// audio thread. The fix is to build the actual tap body in a
// `nonisolated` helper (`makeAudioTapBlock`) and have the
// installTap closure be a single function reference function
// references never carry inferred isolation, so the dispatch
// runtime is happy and the body runs wherever AVAudioEngine
// wants it (the audio thread).
let onMeter: @Sendable (Double) -> Void = { [weak self] meter in
Task { @MainActor [weak self] in
guard let self else { return }
// Lightweight smoothing so the disc ring doesn't jitter.
self.level = self.level * 0.55 + meter * 0.45
}
}
let onSnapshot: @Sendable (AudioBufferSnapshot) -> Void = { [weak self] snapshot in
Task { @MainActor [weak self] in
self?.bufferContinuation?.yield(snapshot)
}
}
let tap = Self.makeAudioTapBlock(
converter: converter,
targetFormat: targetFormat,
hwFormat: hwFormat,
targetSampleRate: targetSampleRate,
onMeter: onMeter,
onSnapshot: onSnapshot
)
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
didInstallTap = true
audioEngine.prepare()
do {
try audioEngine.start()
} catch {
phase = .error(String.localizedStringWithFormat(
NSLocalizedString("preview.error.engineStart", comment: ""),
error.localizedDescription
))
return
}
// 5. Wire up ASR.
let events = asr.transcribe(
stream: stream,
locale: locale
)
asrTask = Task { @MainActor [weak self] in
guard let self else { return }
for await event in events {
switch event {
case .capability:
// Could surface on-device vs cloud here; preview
// doesn't need it.
break
case .partial(let s):
self.currentPartial = s
case .final(let s):
let trimmed = s.trimmingCharacters(in: .whitespacesAndNewlines)
self.lastFinal = trimmed
self.currentPartial = ""
if !trimmed.isEmpty {
self.phase = .idle
} else {
// Empty final: nothing recognized. Return to idle
// without triggering a textbox insert.
self.phase = .idle
}
case .error(let m):
self.errorMessage = m
self.phase = .error(m)
}
}
}
}
// MARK: - Permission helpers (nonisolated)
//
// `SFSpeechRecognizer.requestAuthorization` delivers its callback
// on a TCC reply queue, NOT the main queue. If we wrap that
// callback inline in `start(locale:)` which is `@MainActor`
// Swift 6 strict concurrency infers the closure body as
// `@MainActor`, and the runtime crashes on
// `dispatch_assert_queue` in `_swift_task_checkIsolatedSwift` as
// soon as TCC calls us back.
//
// The first attempt (commit `e8a0310`) extracted the entire
// permission request into a `nonisolated static func` helper.
// That worked in isolation, but the Swift 6 optimizer
// inlined those helpers back into `start(locale:)`. After
// inlining, the `withCheckedContinuation` body and the
// `requestAuthorization` callback were re-typed in the
// `@MainActor` context of the caller, and the runtime
// assertion came right back same crash, different symbol:
// `closure #1 in closure #2 in PreviewASRController.start(locale:)`.
//
// The fix that survives inlining is the *function-reference*
// pattern, the same one used for `installTap` in
// `makeAudioTapBlock` below. The callback is built in a
// `nonisolated` static helper that takes a `CheckedContinuation`
// and returns the `(Status) -> Void` handler. The body of that
// helper has no enclosing actor, so the closure is created in
// nonisolated context. When TCC calls us back, the runtime
// sees a nonisolated closure on a non-main queue and is happy.
//
// `cont.resume(...)` is itself thread-safe on
// `CheckedContinuation`, so we don't need to hop back to the
// main actor before resuming.
private nonisolated static func requestMicrophonePermission() async -> Bool {
// iOS 17+ API; the iOS < 17 fallback (`AVAudioSession.recordPermission`
// + `requestRecordPermission` callback) is gone now that the
// deployment target is iOS 26.
switch AVAudioApplication.shared.recordPermission {
case .granted: return true
case .denied: return false
case .undetermined: return await AVAudioApplication.requestRecordPermission()
@unknown default: return false
}
}
private nonisolated static func requestSpeechRecognitionPermission() async -> Bool {
await withCheckedContinuation { (cont: CheckedContinuation<Bool, Never>) in
SFSpeechRecognizer.requestAuthorization(
Self.makeSpeechAuthHandler(continuation: cont)
)
}
}
private nonisolated static func makeSpeechAuthHandler(
continuation: CheckedContinuation<Bool, Never>
) -> @Sendable (SFSpeechRecognizerAuthorizationStatus) -> Void {
return { status in
continuation.resume(returning: status == .authorized)
}
}
// MARK: - Audio tap (nonisolated, runs on AVAudioEngine render thread)
//
// `AVAudioNode.installTap`'s callback fires on the audio engine's
// real-time render thread. In Swift 6 strict concurrency, a closure
// literal defined inside a `@MainActor` method inherits `@MainActor`
// isolation and `dispatch_assert_queue_fail` fires the moment
// the runtime tries to dispatch that closure on a non-main queue.
//
// The trick is to build the actual tap body in a `nonisolated`
// function and have the installTap closure be a *function reference*
// to that helper. Function references never carry inferred
// isolation, so the dispatch runtime is satisfied and the body
// runs wherever AVAudioEngine wants. State updates to
// `self.level` and the AsyncStream continuation hop back to the
// main actor via `Task { @MainActor in }`, which is itself
// safe to call from a non-isolated context.
private nonisolated static func makeAudioTapBlock(
converter: AVAudioConverter,
targetFormat: AVAudioFormat,
hwFormat: AVAudioFormat,
targetSampleRate: Double,
onMeter: @Sendable @escaping (Double) -> Void,
onSnapshot: @Sendable @escaping (AudioBufferSnapshot) -> Void
) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
// `@Sendable` on the returned closure makes the Sendable
// conformance explicit. `AVAudioNodeTapBlock` is declared as
// a plain escaping closure in the SDK; we cast at the call
// site via `as @Sendable`.
return { buffer, _ in
// Downsample + extract samples + compute RMS in one pass.
let ratio = targetSampleRate / hwFormat.sampleRate
let outCapacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio + 0.5)
guard outCapacity > 0,
let converted = AVAudioPCMBuffer(
pcmFormat: targetFormat,
frameCapacity: outCapacity
) else { return }
var error: NSError?
var supplied = false
converter.convert(to: converted, error: &error) { _, outStatus in
if supplied {
outStatus.pointee = .endOfStream
return nil
}
supplied = true
outStatus.pointee = .haveData
return buffer
}
if error != nil { return }
let n = Int(converted.frameLength)
var samples = [Float](repeating: 0, count: n)
var sumSquares: Float = 0
if let channelData = converted.floatChannelData?[0] {
for i in 0..<n {
let v = channelData[i]
samples[i] = v
sumSquares += v * v
}
}
let rms = n > 0 ? sqrtf(sumSquares / Float(n)) : 0
// RMS for speech is typically 0.02-0.2; the 4x gain pushes
// normal speech into the 0.4-0.8 range for the disc meter.
let meter = min(Double(rms) * 4.0, 1.0)
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: targetSampleRate)
onMeter(meter)
onSnapshot(snapshot)
}
}
}
typealias PreviewASRController = LiveDictationController
+34
View File
@@ -33,6 +33,7 @@ struct SettingsView: View {
if config.engineMode == "cloud" {
promptSection
}
privacySection
resetButton
}
.padding(.horizontal, Spacing.md)
@@ -253,6 +254,39 @@ struct SettingsView: View {
}
}
// MARK: - Privacy
private var privacySection: some View {
VStack(alignment: .leading, spacing: Spacing.xs) {
sectionHeader("settings.privacy.title", subtitle: nil)
VStack(alignment: .leading, spacing: Spacing.sm) {
if let url = LegalLinks.privacyPolicyURL {
Link(destination: url) {
HStack {
Text("settings.privacy.policy")
.font(TypeStyle.body)
.foregroundStyle(palette.accent)
Spacer()
Image(systemName: "arrow.up.right")
.font(.system(size: 12, weight: .semibold))
.foregroundStyle(palette.textTertiary)
}
}
}
VStack(alignment: .leading, spacing: 4) {
Text("settings.privacy.fullAccess.title")
.font(TypeStyle.caption)
.foregroundStyle(palette.textPrimary)
Text("settings.privacy.fullAccess.body")
.font(TypeStyle.caption2)
.foregroundStyle(palette.textSecondary)
.fixedSize(horizontal: false, vertical: true)
}
}
.cardSurface()
}
}
// MARK: - Reset
private var resetButton: some View {
+42 -5
View File
@@ -3,7 +3,21 @@
"OSGKeyboard" = "OSGKeyboard";
/* Onboarding */
"onboarding.welcome.subtitle" = "Hold to talk. Release for polished text, in any app.";
"onboarding.welcome.subtitle" = "Tap to talk. Tap again to finish — polished text in any app.";
"onboarding.enable.fullAccessNote" = "Full Access lets the keyboard reach the microphone and your LLM API. We never log what you type.";
"onboarding.permission.mic.title" = "Microphone access";
"onboarding.permission.mic.body" = "Required for voice input. Audio is processed on your device for transcription.";
"onboarding.permission.mic.allow" = "Allow microphone";
"onboarding.permission.mic.deniedHint" = "Microphone was denied. Open Settings to enable it, or tap Next to continue.";
"onboarding.permission.speech.title" = "Speech recognition";
"onboarding.permission.speech.body" = "On-device speech recognition turns your voice into text. Audio never leaves your device.";
"onboarding.permission.speech.allow" = "Allow speech recognition";
"onboarding.permission.speech.deniedHint" = "Speech recognition was denied. Open Settings to enable it, or tap Next to continue.";
"onboarding.permission.openSettings" = "Open Settings";
"onboarding.permission.status.undetermined" = "Not requested yet";
"onboarding.permission.status.granted" = "Allowed";
"onboarding.permission.status.denied" = "Denied";
"legal.privacyPolicy" = "Privacy Policy";
"onboarding.enable.title" = "Enable OSGKeyboard";
"onboarding.enable.step1" = "Settings → General → Keyboard → Keyboards";
"onboarding.enable.step2" = "Tap “Add New Keyboard…” and choose OSGKeyboard";
@@ -95,6 +109,10 @@
"settings.onDeviceOnly.body" = "Disable cloud fallback. The keyboard reports an error instead of going online.";
"settings.legend.onDevice" = "On-device";
"settings.legend.cloudFallback" = "Cloud fallback";
"settings.privacy.title" = "Privacy";
"settings.privacy.policy" = "Privacy Policy";
"settings.privacy.fullAccess.title" = "About Full Access";
"settings.privacy.fullAccess.body" = "Full Access is required for the microphone and to read your API key. OSGKeyboard does not record or upload what you type with the keyboard.";
/* App group error */
"appGroup.error.title" = "App Group not configured";
@@ -105,8 +123,9 @@
/* Keyboard preview */
"preview.title" = "Keyboard Preview";
"preview.subtitle" = "Tap the disc to start/stop recording. The real keyboard uses the same layout.";
"preview.placeholder" = "Type or tap to record";
"preview.subtitle" = "Tap Start/Stop Recording to test here. On the real keyboard, tap the mic disc to dictate.";
"preview.placeholder" = "Type here or tap the record button";
"preview.placeholder" = "Type here or tap the record button";
"preview.clear" = "Clear text";
"preview.openSettingsA11y" = "Open OSGKeyboard settings";
"preview.modeChip.cycle" = "Cycle input mode";
@@ -118,7 +137,7 @@
"preview.error.engineStart" = "Engine start failed: %@";
/* Keyboard (ext) */
"keyboard.placeholder.idle" = "Hold to talk";
"keyboard.placeholder.idle" = "Tap to talk";
"keyboard.placeholder.preparing" = "Preparing";
"keyboard.placeholder.processing" = "Processing";
"keyboard.placeholder.error" = "Polishing failed";
@@ -129,7 +148,7 @@
"keyboard.denied.speech" = "Speech denied";
"keyboard.openSettingsA11y" = "Open OSGKeyboard settings";
"keyboard.deniedHint" = "Opens the OSGKeyboard settings page where you can grant microphone or speech recognition access.";
"keyboard.pressToTalkA11y" = "Push to talk";
"keyboard.tapToTalkA11y" = "Tap to talk";
/* Mode chip labels (used in both ext + preview stub) */
"mode.off" = "Off";
@@ -146,3 +165,21 @@
"locale.en-US" = "English (US)";
"locale.ja-JP" = "Japanese";
"locale.ko-KR" = "Korean";
/* Flow session */
"flow.error.noSpeech" = "No speech detected. Please try again.";
"keyboard.denied.mic" = "Microphone access denied";
"keyboard.denied.speech" = "Speech recognition denied";
/* Home · Flow session */
"home.flow.active" = "Voice session active";
"home.flow.inactive" = "Voice session inactive";
"home.flow.hint" = "Voice session starts automatically. Switch to any app and tap the keyboard mic to dictate.";
"home.flow.starting" = "Starting voice session…";
"home.flow.openSettings" = "Open Settings to grant permissions";
"home.flow.start" = "Start voice session";
"home.flow.end" = "End voice session";
"flow.error.speechRequired" = "Speech recognition access is required for voice sessions.";
"flow.error.micRequired" = "Microphone access is required for background voice sessions.";
"flow.error.micUnavailable" = "Microphone is unavailable on this device.";
"flow.error.audioUnavailable" = "Could not start background audio.";
+39 -3
View File
@@ -3,7 +3,21 @@
"OSGKeyboard" = "OSGKeyboard";
/* Onboarding */
"onboarding.welcome.subtitle" = "按住说话,松开即得润色文字。";
"onboarding.welcome.subtitle" = "点按说话,再点结束 — 任意 App 里都能获得润色文字。";
"onboarding.enable.fullAccessNote" = "完全访问用于麦克风与 API 配置读取。我们不会记录或上传你的击键内容。";
"onboarding.permission.mic.title" = "麦克风权限";
"onboarding.permission.mic.body" = "语音输入需要麦克风。音频在设备端转录,不会上传原始录音。";
"onboarding.permission.mic.allow" = "允许麦克风";
"onboarding.permission.mic.deniedHint" = "麦克风被拒绝。可前往设置开启,或点「下一步」继续。";
"onboarding.permission.speech.title" = "语音识别权限";
"onboarding.permission.speech.body" = "端侧语音识别将语音转为文字。音频不会离开你的设备。";
"onboarding.permission.speech.allow" = "允许语音识别";
"onboarding.permission.speech.deniedHint" = "语音识别被拒绝。可前往设置开启,或点「下一步」继续。";
"onboarding.permission.openSettings" = "打开设置";
"onboarding.permission.status.undetermined" = "尚未请求";
"onboarding.permission.status.granted" = "已允许";
"onboarding.permission.status.denied" = "已拒绝";
"legal.privacyPolicy" = "隐私政策";
"onboarding.enable.title" = "启用 OSGKeyboard";
"onboarding.enable.step1" = "设置 → 通用 → 键盘 → 键盘";
"onboarding.enable.step2" = "点击「添加新键盘…」并选择 OSGKeyboard";
@@ -95,6 +109,10 @@
"settings.onDeviceOnly.body" = "禁用云端回退,识别失败时会报错而非联网。";
"settings.legend.onDevice" = "端侧识别";
"settings.legend.cloudFallback" = "需联网";
"settings.privacy.title" = "隐私";
"settings.privacy.policy" = "隐私政策";
"settings.privacy.fullAccess.title" = "关于完全访问";
"settings.privacy.fullAccess.body" = "完全访问用于麦克风与读取 API Key。OSGKeyboard 不会记录或上传你在键盘上的击键内容。";
/* App group error */
"appGroup.error.title" = "App Group 未配置";
@@ -105,8 +123,8 @@
/* Keyboard preview */
"preview.title" = "键盘预览";
"preview.subtitle" = "点按 disc 开始/结束录音;真实键盘使用同样布局。";
"preview.placeholder" = "试着输入或按 disc 录音";
"preview.subtitle" = "点击「开始录音」/「停止录音」按钮测试;真实键盘为长按麦克风圆盘。";
"preview.placeholder" = "试着输入或点击按钮录音";
"preview.clear" = "清空";
"preview.openSettingsA11y" = "打开 OSGKeyboard 设置";
"preview.modeChip.cycle" = "切换输入模式";
@@ -146,3 +164,21 @@
"locale.en-US" = "English (US)";
"locale.ja-JP" = "日本語";
"locale.ko-KR" = "한국어";
/* Flow session */
"flow.error.noSpeech" = "未检测到语音,请重试。";
"keyboard.denied.mic" = "麦克风权限被拒绝";
"keyboard.denied.speech" = "语音识别权限被拒绝";
/* Home · Flow session */
"home.flow.active" = "语音会话进行中";
"home.flow.inactive" = "语音会话未启动";
"home.flow.hint" = "语音会话会自动启动。切到任意 App,点按键盘麦克风即可说话。";
"home.flow.starting" = "正在启动语音会话…";
"home.flow.openSettings" = "前往设置授予权限";
"home.flow.start" = "启动语音会话";
"home.flow.end" = "结束语音会话";
"flow.error.speechRequired" = "需要语音识别权限才能使用语音会话。";
"flow.error.micRequired" = "需要麦克风权限才能维持后台语音会话。";
"flow.error.micUnavailable" = "当前设备无法使用麦克风。";
"flow.error.audioUnavailable" = "无法启动后台音频。";