feat: flow session reliability, keyboard UX, and audio pre-roll fix

Renew voice sessions while the host app stays foreground, auto-start flow from the keyboard with a Start action, and prevent leading audio loss via pre-roll buffering and faster signal polling.
This commit is contained in:
Rocky
2026-06-20 00:57:16 +08:00
parent 38a36ed504
commit db5151ad9a
12 changed files with 196 additions and 77 deletions
+4
View File
@@ -35,6 +35,9 @@ struct OSGKeyboardApp: App {
} }
} }
.environmentObject(flowManager) .environmentObject(flowManager)
.onAppear {
flowManager.setAppForeground(scenePhase == .active)
}
.onOpenURL { url in .onOpenURL { url in
guard url.scheme == "osgkeyboard" else { return } guard url.scheme == "osgkeyboard" else { return }
switch url.host { switch url.host {
@@ -56,6 +59,7 @@ struct OSGKeyboardApp: App {
if done { flowManager.autoStartIfNeeded() } if done { flowManager.autoStartIfNeeded() }
} }
.onChange(of: scenePhase) { _, phase in .onChange(of: scenePhase) { _, phase in
flowManager.setAppForeground(phase == .active)
guard phase == .active, AppGroup.isAvailable, config.hasCompletedOnboarding else { return } guard phase == .active, AppGroup.isAvailable, config.hasCompletedOnboarding else { return }
if flowManager.isActive { if flowManager.isActive {
flowManager.extendSession() flowManager.extendSession()
+22 -1
View File
@@ -35,6 +35,8 @@ final class FlowSessionManager: ObservableObject {
private var asrTask: Task<Void, Never>? private var asrTask: Task<Void, Never>?
private var currentPartial = "" private var currentPartial = ""
private var lastFinal = "" private var lastFinal = ""
/// True while the host app scene is `.active` drives foreground renewal.
private var isAppForeground = false
init() { init() {
Task { @MainActor [weak self] in Task { @MainActor [weak self] in
@@ -169,6 +171,24 @@ final class FlowSessionManager: ObservableObject {
scheduleExpiry(after: duration) scheduleExpiry(after: duration)
} }
/// Called from `OSGKeyboardApp` when `scenePhase` changes.
func setAppForeground(_ foreground: Bool) {
isAppForeground = foreground
if foreground, isActive {
renewSessionIfNeededWhileForeground()
}
}
/// Extend the session before it expires while the host app stays in foreground.
private func renewSessionIfNeededWhileForeground() {
guard isActive, isAppForeground else { return }
guard let remaining = FlowSessionBridge.remainingSessionDuration() else { return }
let threshold = FlowSessionKeys.defaultSessionDuration * 0.25
guard remaining < threshold else { return }
extendSession()
debug("Flow session renewed in foreground (\(Int(threshold))s threshold)")
}
// MARK: - Session start // MARK: - Session start
private func startSessionAsync(duration: TimeInterval) async { private func startSessionAsync(duration: TimeInterval) async {
@@ -217,7 +237,7 @@ final class FlowSessionManager: ObservableObject {
pollingTask = Task { @MainActor [weak self] in pollingTask = Task { @MainActor [weak self] in
while !Task.isCancelled { while !Task.isCancelled {
self?.handleKeyboardSignal() self?.handleKeyboardSignal()
try? await Task.sleep(nanoseconds: 100_000_000) try? await Task.sleep(nanoseconds: 50_000_000)
} }
} }
} }
@@ -424,6 +444,7 @@ final class FlowSessionManager: ObservableObject {
heartbeatTask = Task { @MainActor [weak self] in heartbeatTask = Task { @MainActor [weak self] in
while !Task.isCancelled { while !Task.isCancelled {
FlowSessionBridge.writeHeartbeat() FlowSessionBridge.writeHeartbeat()
self?.renewSessionIfNeededWhileForeground()
try? await Task.sleep(nanoseconds: 1_000_000_000) try? await Task.sleep(nanoseconds: 1_000_000_000)
guard self?.isActive == true else { break } guard self?.isActive == true else { break }
} }
+3 -22
View File
@@ -42,8 +42,8 @@ struct HomeView: View {
engineStatusLine engineStatusLine
.padding(.horizontal, Spacing.lg) .padding(.horizontal, Spacing.lg)
.padding(.top, Spacing.md) .padding(.top, Spacing.xl)
.padding(.bottom, Spacing.xs) .padding(.bottom, Spacing.sm)
} }
.frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top) .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top)
} }
@@ -81,7 +81,7 @@ struct HomeView: View {
Image("osglogo") Image("osglogo")
.resizable() .resizable()
.scaledToFit() .scaledToFit()
.frame(width: 120, height: 34) .frame(width: 144, height: 41)
.accessibilityHidden(true) .accessibilityHidden(true)
statusCapsule statusCapsule
@@ -92,13 +92,6 @@ struct HomeView: View {
private var statusCapsule: some View { private var statusCapsule: some View {
HStack(spacing: Spacing.sm) { HStack(spacing: Spacing.sm) {
Text(statusLine)
.font(TypeStyle.status)
.foregroundStyle(palette.textSecondary)
.fixedSize()
capsuleDivider
flowCapsuleSegment flowCapsuleSegment
if flowManager.isActive { if flowManager.isActive {
@@ -125,12 +118,6 @@ struct HomeView: View {
.animation(Motion.soft, value: flowManager.isActive) .animation(Motion.soft, value: flowManager.isActive)
} }
private var capsuleDivider: some View {
Circle()
.fill(palette.dividerStrong)
.frame(width: 3, height: 3)
}
@ViewBuilder @ViewBuilder
private var flowCapsuleSegment: some View { private var flowCapsuleSegment: some View {
HStack(spacing: Spacing.xs) { HStack(spacing: Spacing.xs) {
@@ -165,12 +152,6 @@ struct HomeView: View {
: palette.surface.opacity(0.88) : palette.surface.opacity(0.88)
} }
private var statusLine: String {
config.isConfigured
? NSLocalizedString("home.status.ready", comment: "")
: NSLocalizedString("home.status.setupIncomplete", comment: "")
}
// MARK: - Flow extras (warnings / hint) // MARK: - Flow extras (warnings / hint)
@ViewBuilder @ViewBuilder
+17 -7
View File
@@ -184,6 +184,11 @@ private struct OnboardingHeroIcon: View {
} }
} }
private enum OnboardingLayoutMetrics {
/// Matches welcome page: logo tagline, and subtitle next block.
static let heroTextGap: CGFloat = Spacing.hero
}
// MARK: - Welcome // MARK: - Welcome
private struct WelcomePage: View { private struct WelcomePage: View {
@@ -209,11 +214,11 @@ private struct WelcomePage: View {
} }
.multilineTextAlignment(.center) .multilineTextAlignment(.center)
.padding(.horizontal, Spacing.xl) .padding(.horizontal, Spacing.xl)
.padding(.top, Spacing.hero) .padding(.top, OnboardingLayoutMetrics.heroTextGap)
PrivacyFootnote() PrivacyFootnote()
.padding(.horizontal, Spacing.xl) .padding(.horizontal, Spacing.xl)
.padding(.top, Spacing.hero) .padding(.top, OnboardingLayoutMetrics.heroTextGap)
if let url = LegalLinks.privacyPolicyURL { if let url = LegalLinks.privacyPolicyURL {
Link(destination: url) { Link(destination: url) {
@@ -388,12 +393,12 @@ private struct PermissionPageLayout: View {
var deniedHint: LocalizedStringKey? = nil var deniedHint: LocalizedStringKey? = nil
var body: some View { var body: some View {
VStack(spacing: Spacing.xl) { VStack(spacing: 0) {
Spacer() Spacer()
OnboardingHeroIcon(systemName: icon, circleSize: 96, iconSize: 40) OnboardingHeroIcon(systemName: icon, circleSize: 96, iconSize: 40)
VStack(spacing: Spacing.sm) { VStack(spacing: Spacing.xs) {
Text(title) Text(title)
.font(TypeStyle.title2) .font(TypeStyle.title2)
.foregroundStyle(palette.textPrimary) .foregroundStyle(palette.textPrimary)
@@ -403,6 +408,7 @@ private struct PermissionPageLayout: View {
.multilineTextAlignment(.center) .multilineTextAlignment(.center)
.padding(.horizontal, Spacing.xl) .padding(.horizontal, Spacing.xl)
} }
.padding(.top, OnboardingLayoutMetrics.heroTextGap)
HStack(spacing: 6) { HStack(spacing: 6) {
Circle().fill(statusColor).frame(width: 8, height: 8) Circle().fill(statusColor).frame(width: 8, height: 8)
@@ -410,6 +416,7 @@ private struct PermissionPageLayout: View {
.font(TypeStyle.caption) .font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary) .foregroundStyle(palette.textSecondary)
} }
.padding(.top, OnboardingLayoutMetrics.heroTextGap)
if let deniedHint { if let deniedHint {
Text(deniedHint) Text(deniedHint)
@@ -417,6 +424,7 @@ private struct PermissionPageLayout: View {
.foregroundStyle(palette.warning) .foregroundStyle(palette.warning)
.multilineTextAlignment(.center) .multilineTextAlignment(.center)
.padding(.horizontal, Spacing.xl) .padding(.horizontal, Spacing.xl)
.padding(.top, Spacing.md)
} }
VStack(spacing: Spacing.sm) { VStack(spacing: Spacing.sm) {
@@ -436,6 +444,7 @@ private struct PermissionPageLayout: View {
} }
} }
.padding(.horizontal, Spacing.lg) .padding(.horizontal, Spacing.lg)
.padding(.top, Spacing.xl)
Spacer() Spacer()
} }
@@ -448,12 +457,12 @@ private struct EnableKeyboardPage: View {
@Environment(\.themePalette) private var palette: ThemePalette @Environment(\.themePalette) private var palette: ThemePalette
var body: some View { var body: some View {
VStack(spacing: Spacing.xl) { VStack(spacing: 0) {
Spacer() Spacer()
OnboardingHeroIcon(systemName: "keyboard.fill", circleSize: 96, iconSize: 40) OnboardingHeroIcon(systemName: "keyboard.fill", circleSize: 96, iconSize: 40)
VStack(spacing: Spacing.sm) { VStack(spacing: Spacing.xs) {
Text("onboarding.enable.title") Text("onboarding.enable.title")
.font(TypeStyle.title2) .font(TypeStyle.title2)
.foregroundStyle(palette.textPrimary) .foregroundStyle(palette.textPrimary)
@@ -463,6 +472,7 @@ private struct EnableKeyboardPage: View {
.multilineTextAlignment(.center) .multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg) .padding(.horizontal, Spacing.lg)
} }
.padding(.top, OnboardingLayoutMetrics.heroTextGap)
VStack(alignment: .leading, spacing: Spacing.lg) { VStack(alignment: .leading, spacing: Spacing.lg) {
step(num: 1, text: NSLocalizedString("onboarding.enable.step1", comment: "")) step(num: 1, text: NSLocalizedString("onboarding.enable.step1", comment: ""))
@@ -472,7 +482,7 @@ private struct EnableKeyboardPage: View {
} }
.frame(maxWidth: .infinity, alignment: .leading) .frame(maxWidth: .infinity, alignment: .leading)
.padding(.horizontal, Spacing.xl) .padding(.horizontal, Spacing.xl)
.padding(.top, Spacing.sm) .padding(.top, OnboardingLayoutMetrics.heroTextGap)
Button { Button {
AppPermissions.openSystemSettings() AppPermissions.openSystemSettings()
+8 -8
View File
@@ -7,11 +7,11 @@
"onboarding.welcome.subtitle" = "Your open-source voice keyboard"; "onboarding.welcome.subtitle" = "Your open-source voice keyboard";
"onboarding.enable.fullAccessNote" = "Full Access lets the keyboard reach the microphone and your LLM API. We never log what you type."; "onboarding.enable.fullAccessNote" = "Full Access lets the keyboard reach the microphone and your LLM API. We never log what you type.";
"onboarding.permission.mic.title" = "Microphone access"; "onboarding.permission.mic.title" = "Microphone access";
"onboarding.permission.mic.body" = "Required for voice input. Audio is processed on your device for transcription."; "onboarding.permission.mic.body" = "Voice input requires your permission to use the microphone.";
"onboarding.permission.mic.allow" = "Allow microphone"; "onboarding.permission.mic.allow" = "Allow microphone";
"onboarding.permission.mic.deniedHint" = "Microphone was denied. Open Settings to enable it, or tap Next to continue."; "onboarding.permission.mic.deniedHint" = "Microphone was denied. Open Settings to enable it, or tap Next to continue.";
"onboarding.permission.speech.title" = "Speech recognition"; "onboarding.permission.speech.title" = "Speech recognition";
"onboarding.permission.speech.body" = "On-device speech recognition turns your voice into text. Audio never leaves your device."; "onboarding.permission.speech.body" = "On-device speech recognition turns your voice into text.";
"onboarding.permission.speech.allow" = "Allow speech recognition"; "onboarding.permission.speech.allow" = "Allow speech recognition";
"onboarding.permission.speech.deniedHint" = "Speech recognition was denied. Open Settings to enable it, or tap Next to continue."; "onboarding.permission.speech.deniedHint" = "Speech recognition was denied. Open Settings to enable it, or tap Next to continue.";
"onboarding.permission.openSettings" = "Open Settings"; "onboarding.permission.openSettings" = "Open Settings";
@@ -43,12 +43,12 @@
"common.newline" = "Return"; "common.newline" = "Return";
/* Privacy footnote (onboarding welcome page) */ /* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "Audio stays on device"; "privacy.audio.title" = "On-device transcription";
"privacy.audio.body" = "Transcribed locally with Apples speech engine."; "privacy.audio.body" = "Powered by the on-device iOS speech engine.";
"privacy.network.title" = "Only the polished text is sent"; "privacy.network.title" = "Automatic text polish";
"privacy.network.body" = "Sent to your chosen LLM to add structure & punctuation."; "privacy.network.body" = "AI formats and polishes your text automatically.";
"privacy.universal.title" = "Works everywhere"; "privacy.universal.title" = "Works everywhere";
"privacy.universal.body" = "WeChat, Notes, Mail, ChatGPT, Claude, Cursor — anywhere a keyboard appears."; "privacy.universal.body" = "Chat, email, vibe coding — just speak.";
/* Home */ /* Home */
"home.status.ready" = "Ready"; "home.status.ready" = "Ready";
@@ -184,7 +184,7 @@
/* Home · Flow session */ /* Home · Flow session */
"home.flow.active" = "Voice session active"; "home.flow.active" = "Voice session active";
"home.flow.label" = "Voice session"; "home.flow.label" = "Ready";
"home.flow.inactive" = "Voice session inactive"; "home.flow.inactive" = "Voice session inactive";
"home.flow.hint" = "Voice session starts automatically. Switch to any app and tap the keyboard mic to dictate."; "home.flow.hint" = "Voice session starts automatically. Switch to any app and tap the keyboard mic to dictate.";
"home.flow.starting" = "Starting voice session…"; "home.flow.starting" = "Starting voice session…";
@@ -7,11 +7,11 @@
"onboarding.welcome.subtitle" = "你的开源语音输入法"; "onboarding.welcome.subtitle" = "你的开源语音输入法";
"onboarding.enable.fullAccessNote" = "完全访问用于麦克风与 API 配置读取。我们不会记录或上传你的击键内容。"; "onboarding.enable.fullAccessNote" = "完全访问用于麦克风与 API 配置读取。我们不会记录或上传你的击键内容。";
"onboarding.permission.mic.title" = "麦克风权限"; "onboarding.permission.mic.title" = "麦克风权限";
"onboarding.permission.mic.body" = "语音输入需要麦克风。音频在设备端转录,不会上传原始录音。"; "onboarding.permission.mic.body" = "语音输入需要您授予麦克风权限";
"onboarding.permission.mic.allow" = "允许麦克风"; "onboarding.permission.mic.allow" = "允许麦克风";
"onboarding.permission.mic.deniedHint" = "麦克风被拒绝。可前往设置开启,或点「下一步」继续。"; "onboarding.permission.mic.deniedHint" = "麦克风被拒绝。可前往设置开启,或点「下一步」继续。";
"onboarding.permission.speech.title" = "语音识别权限"; "onboarding.permission.speech.title" = "语音识别权限";
"onboarding.permission.speech.body" = "端侧语音识别将语音转为文字。音频不会离开你的设备。"; "onboarding.permission.speech.body" = "端侧语音识别将语音转为文字。";
"onboarding.permission.speech.allow" = "允许语音识别"; "onboarding.permission.speech.allow" = "允许语音识别";
"onboarding.permission.speech.deniedHint" = "语音识别被拒绝。可前往设置开启,或点「下一步」继续。"; "onboarding.permission.speech.deniedHint" = "语音识别被拒绝。可前往设置开启,或点「下一步」继续。";
"onboarding.permission.openSettings" = "打开设置"; "onboarding.permission.openSettings" = "打开设置";
@@ -43,12 +43,12 @@
"common.newline" = "换行"; "common.newline" = "换行";
/* Privacy footnote (onboarding welcome page) */ /* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "音频不出本机"; "privacy.audio.title" = "支持本地转写";
"privacy.audio.body" = "由 Apple 端侧引擎转录"; "privacy.audio.body" = "支持 iOS 本地引擎转录";
"privacy.network.title" = "仅发送润色后文字"; "privacy.network.title" = "文字自动润色";
"privacy.network.body" = "仅向所选 LLM 发送润色后文字,用于整理结构和标点。"; "privacy.network.body" = "通过 AI 自动对文字进行排版润色";
"privacy.universal.title" = "处处可用"; "privacy.universal.title" = "处处可用";
"privacy.universal.body" = "微信、备忘录、邮件、ChatGPT、Claude、Cursor — 任何键盘出现的地方。"; "privacy.universal.body" = "聊天、邮件、Vibe Coding 动嘴就行";
/* Home */ /* Home */
"home.status.ready" = "就绪"; "home.status.ready" = "就绪";
@@ -183,7 +183,7 @@
/* Home · Flow session */ /* Home · Flow session */
"home.flow.active" = "语音会话进行中"; "home.flow.active" = "语音会话进行中";
"home.flow.label" = "语音会话"; "home.flow.label" = "就绪";
"home.flow.inactive" = "语音会话未启动"; "home.flow.inactive" = "语音会话未启动";
"home.flow.hint" = "语音会话会自动启动。切到任意 App,点按键盘麦克风即可说话。"; "home.flow.hint" = "语音会话会自动启动。切到任意 App,点按键盘麦克风即可说话。";
"home.flow.starting" = "正在启动语音会话…"; "home.flow.starting" = "正在启动语音会话…";
@@ -66,6 +66,8 @@ public final class KeyboardViewController: UIInputViewController {
private var flowSessionMonitorTask: Task<Void, Never>? private var flowSessionMonitorTask: Task<Void, Never>?
private var flowSessionDarwinObserver: FlowSessionDarwinObserver? private var flowSessionDarwinObserver: FlowSessionDarwinObserver?
private var isAwaitingFlowResult = false private var isAwaitingFlowResult = false
private var lastFlowAutoStartAttempt: TimeInterval = 0
private static let flowAutoStartCooldown: TimeInterval = 20
// MARK: - Lifecycle // MARK: - Lifecycle
@@ -121,6 +123,7 @@ public final class KeyboardViewController: UIInputViewController {
state.endRecording = { [weak self] in self?.pressEnded() } state.endRecording = { [weak self] in self?.pressEnded() }
state.tapMic = { [weak self] in self?.toggleRecording() } state.tapMic = { [weak self] in self?.toggleRecording() }
state.openSettings = { [weak self] in self?.openHostApp() } state.openSettings = { [weak self] in self?.openHostApp() }
state.startFlowSession = { [weak self] in self?.beginFlowStart() }
state.setMode = { [weak self] m in self?.persistMode(m) } state.setMode = { [weak self] m in self?.persistMode(m) }
state.setLocale = { [weak self] l in self?.persistLocale(l) } state.setLocale = { [weak self] l in self?.persistLocale(l) }
state.setEngineMode = { [weak self] m in self?.persistEngineMode(m) } state.setEngineMode = { [weak self] m in self?.persistEngineMode(m) }
@@ -199,6 +202,24 @@ public final class KeyboardViewController: UIInputViewController {
} }
} }
wasFlowSessionActive = active wasFlowSessionActive = active
if !active {
maybeAutoStartFlowSession()
}
}
/// When the host session is down, proactively jump to the app to start it.
private func maybeAutoStartFlowSession() {
guard !FlowSessionBridge.isSessionActive() else { return }
guard !isPendingFlowStart, !isFlowRecording, !isAwaitingFlowResult else { return }
guard state.mode != .off else { return }
guard hasFullAccess, AppGroup.isAvailable else { return }
guard case .idle = state.phase else { return }
let now = Date().timeIntervalSince1970
guard now - lastFlowAutoStartAttempt >= Self.flowAutoStartCooldown else { return }
lastFlowAutoStartAttempt = now
beginFlowStart()
} }
private func showFlowSessionExpiredHint() { private func showFlowSessionExpiredHint() {
@@ -303,6 +324,7 @@ public final class KeyboardViewController: UIInputViewController {
} }
private func beginFlowStart() { private func beginFlowStart() {
guard !isPendingFlowStart else { return }
isPendingFlowStart = true isPendingFlowStart = true
isFlowRecording = false isFlowRecording = false
flowStartDeadline = Date().timeIntervalSince1970 + FlowWatchdog.startTimeout flowStartDeadline = Date().timeIntervalSince1970 + FlowWatchdog.startTimeout
+45 -15
View File
@@ -19,7 +19,13 @@ import SwiftUI
import OSGKeyboardShared import OSGKeyboardShared
private enum KeyboardLayoutMetrics { private enum KeyboardLayoutMetrics {
static let sideActionButtonSize: CGFloat = 44 static let sideActionButtonSize: CGFloat = 53
static let sideActionIconSize: CGFloat = 19
static let sideSpaceBarWidth: CGFloat = 19
static let micFlankMinSpacing: CGFloat = 36
static let sideActionStackSpacing: CGFloat = 16
/// Outer inset for delete / return·space from screen edges (8 pt 24 pt, +200%).
static let sideActionHorizontalInset: CGFloat = Spacing.xs * 3
} }
public struct KeyboardRootView: View { public struct KeyboardRootView: View {
@@ -36,7 +42,6 @@ public struct KeyboardRootView: View {
/// it up. /// it up.
static let totalHeight: CGFloat = 280 static let totalHeight: CGFloat = 280
private static let topBarHeight: CGFloat = 38 private static let topBarHeight: CGFloat = 38
private static let sideActionStackSpacing: CGFloat = 10
private var palette: ThemePalette { private var palette: ThemePalette {
colorScheme == .dark ? Palette.dark : Palette.light colorScheme == .dark ? Palette.dark : Palette.light
@@ -52,14 +57,16 @@ public struct KeyboardRootView: View {
} }
.padding(.top, 4) .padding(.top, 4)
.padding(.bottom, 6) .padding(.bottom, 6)
// Let the system UI chrome show through by drawing no background .background(keyboardBackground)
// of our own.
.background(Color.clear)
.frame(height: Self.totalHeight) .frame(height: Self.totalHeight)
// Feed the resolved palette to all nested chips/buttons. // Feed the resolved palette to all nested chips/buttons.
.environment(\.themePalette, palette) .environment(\.themePalette, palette)
} }
private var keyboardBackground: Color {
colorScheme == .dark ? palette.background : Color.clear
}
// MARK: - Top bar // MARK: - Top bar
private var topBar: some View { private var topBar: some View {
@@ -99,7 +106,8 @@ public struct KeyboardRootView: View {
phase: state.phase, phase: state.phase,
transcript: state.lastTranscript, transcript: state.lastTranscript,
flowSessionActive: state.flowSessionActive, flowSessionActive: state.flowSessionActive,
openSettings: state.openSettings openSettings: state.openSettings,
startFlowSession: state.startFlowSession
) )
.frame(height: 22) .frame(height: 22)
@@ -117,11 +125,13 @@ public struct KeyboardRootView: View {
/// HStack vertical alignment keeps delete, mic centre, and the gap /// HStack vertical alignment keeps delete, mic centre, and the gap
/// between return/space on one horizontal axis. /// between return/space on one horizontal axis.
private var micActionRow: some View { private var micActionRow: some View {
HStack(alignment: .center, spacing: 20) { HStack(alignment: .center, spacing: 0) {
CircularToolbarButton(systemName: "delete.left", label: "delete") { CircularToolbarButton(systemName: "delete.left", label: "delete") {
state.deleteBackward() state.deleteBackward()
} }
Spacer(minLength: KeyboardLayoutMetrics.micFlankMinSpacing)
RecordButton( RecordButton(
phase: buttonPhase, phase: buttonPhase,
level: state.level, level: state.level,
@@ -130,7 +140,9 @@ public struct KeyboardRootView: View {
) )
.frame(width: 132, height: 132) .frame(width: 132, height: 132)
VStack(spacing: Self.sideActionStackSpacing) { Spacer(minLength: KeyboardLayoutMetrics.micFlankMinSpacing)
VStack(spacing: KeyboardLayoutMetrics.sideActionStackSpacing) {
CircularToolbarButton(systemName: "return", label: "newline") { CircularToolbarButton(systemName: "return", label: "newline") {
state.insertNewline() state.insertNewline()
} }
@@ -139,6 +151,7 @@ public struct KeyboardRootView: View {
} }
} }
} }
.padding(.horizontal, KeyboardLayoutMetrics.sideActionHorizontalInset)
.frame(maxWidth: .infinity) .frame(maxWidth: .infinity)
} }
@@ -191,6 +204,7 @@ private struct TranscriptLine: View {
let transcript: String let transcript: String
let flowSessionActive: Bool let flowSessionActive: Bool
let openSettings: () -> Void let openSettings: () -> Void
let startFlowSession: () -> Void
var body: some View { var body: some View {
ZStack { ZStack {
@@ -201,9 +215,18 @@ private struct TranscriptLine: View {
.font(TypeStyle.caption) .font(TypeStyle.caption)
.foregroundStyle(palette.textTertiary) .foregroundStyle(palette.textTertiary)
} else { } else {
ExtL10n.text("keyboard.flow.sessionInactive") HStack(spacing: 6) {
.font(TypeStyle.caption) ExtL10n.text("keyboard.flow.sessionInactive")
.foregroundStyle(palette.textTertiary) .font(TypeStyle.caption)
.foregroundStyle(palette.textTertiary)
Button(action: startFlowSession) {
ExtL10n.text("keyboard.flow.start")
.font(TypeStyle.caption)
.foregroundStyle(palette.accent)
}
.buttonStyle(.plain)
.accessibilityHint(ExtL10n.text("keyboard.flow.startA11y"))
}
} }
case .requestingPermissions: case .requestingPermissions:
HStack(spacing: 6) { HStack(spacing: 6) {
@@ -267,6 +290,7 @@ private struct TranscriptLine: View {
// MARK: - Circular toolbar button // MARK: - Circular toolbar button
private struct CircularToolbarButton: View { private struct CircularToolbarButton: View {
@Environment(\.colorScheme) private var colorScheme
@Environment(\.themePalette) private var palette: ThemePalette @Environment(\.themePalette) private var palette: ThemePalette
let systemName: String? let systemName: String?
@@ -294,20 +318,26 @@ private struct CircularToolbarButton: View {
if spaceStyle { if spaceStyle {
Capsule() Capsule()
.fill(palette.textPrimary) .fill(palette.textPrimary)
.frame(width: 16, height: 3) .frame(width: KeyboardLayoutMetrics.sideSpaceBarWidth, height: 3)
} else if let systemName { } else if let systemName {
Image(systemName: systemName) Image(systemName: systemName)
.font(.system(size: 16, weight: .medium)) .font(.system(size: KeyboardLayoutMetrics.sideActionIconSize, weight: .medium))
.foregroundStyle(palette.textPrimary) .foregroundStyle(palette.textPrimary)
} }
} }
.frame(width: KeyboardLayoutMetrics.sideActionButtonSize, height: KeyboardLayoutMetrics.sideActionButtonSize) .frame(width: KeyboardLayoutMetrics.sideActionButtonSize, height: KeyboardLayoutMetrics.sideActionButtonSize)
.background(palette.surfaceElevated, in: Circle()) .background(sideButtonFill, in: Circle())
.overlay(Circle().stroke(palette.divider, lineWidth: 0.5)) .overlay(Circle().stroke(palette.dividerStrong, lineWidth: 0.5))
} }
.buttonStyle(.plain) .buttonStyle(.plain)
.accessibilityLabel(Text(label)) .accessibilityLabel(Text(label))
} }
private var sideButtonFill: Color {
colorScheme == .dark
? Color(red: 0.20, green: 0.20, blue: 0.22)
: palette.surfaceElevated
}
} }
// MARK: - Status badge // MARK: - Status badge
+8 -6
View File
@@ -27,12 +27,12 @@
"common.newline" = "Return"; "common.newline" = "Return";
/* Privacy footnote (onboarding welcome page) */ /* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "Audio stays on device"; "privacy.audio.title" = "On-device transcription";
"privacy.audio.body" = "Transcribed locally with Apples speech engine."; "privacy.audio.body" = "Powered by the on-device iOS speech engine.";
"privacy.network.title" = "Only the polished text is sent"; "privacy.network.title" = "Automatic text polish";
"privacy.network.body" = "Sent to your chosen LLM to add structure & punctuation."; "privacy.network.body" = "AI formats and polishes your text automatically.";
"privacy.universal.title" = "Works everywhere"; "privacy.universal.title" = "Works everywhere";
"privacy.universal.body" = "WeChat, Notes, Mail, ChatGPT, Claude, Cursor — anywhere a keyboard appears."; "privacy.universal.body" = "Chat, email, vibe coding — just speak.";
/* Home */ /* Home */
"home.status.ready" = "Ready"; "home.status.ready" = "Ready";
@@ -127,7 +127,9 @@
"keyboard.tapToTalkA11y" = "Tap to talk"; "keyboard.tapToTalkA11y" = "Tap to talk";
/* Flow session (keyboard) */ /* Flow session (keyboard) */
"keyboard.flow.sessionInactive" = "Open OSGKeyboard to start voice session"; "keyboard.flow.sessionInactive" = "Voice session off";
"keyboard.flow.start" = "Start";
"keyboard.flow.startA11y" = "Start voice session";
"keyboard.flow.sessionExpired" = "Voice session ended. Open OSGKeyboard to restart."; "keyboard.flow.sessionExpired" = "Voice session ended. Open OSGKeyboard to restart.";
"keyboard.flow.startingSession" = "Starting voice session…"; "keyboard.flow.startingSession" = "Starting voice session…";
"keyboard.flow.transcribing" = "Transcribing…"; "keyboard.flow.transcribing" = "Transcribing…";
@@ -27,12 +27,12 @@
"common.newline" = "换行"; "common.newline" = "换行";
/* Privacy footnote (onboarding welcome page) */ /* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "音频不出本机"; "privacy.audio.title" = "支持本地转写";
"privacy.audio.body" = "由 Apple 端侧引擎转录"; "privacy.audio.body" = "支持 iOS 本地引擎转录";
"privacy.network.title" = "仅发送润色后文字"; "privacy.network.title" = "文字自动润色";
"privacy.network.body" = "仅向所选 LLM 发送润色后文字,用于整理结构和标点。"; "privacy.network.body" = "通过 AI 自动对文字进行排版润色";
"privacy.universal.title" = "处处可用"; "privacy.universal.title" = "处处可用";
"privacy.universal.body" = "微信、备忘录、邮件、ChatGPT、Claude、Cursor — 任何键盘出现的地方。"; "privacy.universal.body" = "聊天、邮件、Vibe Coding 动嘴就行";
/* Home */ /* Home */
"home.status.ready" = "就绪"; "home.status.ready" = "就绪";
@@ -127,7 +127,9 @@
"keyboard.tapToTalkA11y" = "点按说话"; "keyboard.tapToTalkA11y" = "点按说话";
/* Flow session (keyboard) */ /* Flow session (keyboard) */
"keyboard.flow.sessionInactive" = "请打开 OSGKeyboard 启动语音会话"; "keyboard.flow.sessionInactive" = "语音会话未启动";
"keyboard.flow.start" = "启动";
"keyboard.flow.startA11y" = "启动语音会话";
"keyboard.flow.sessionExpired" = "语音会话已结束,请打开 OSGKeyboard 重新启动"; "keyboard.flow.sessionExpired" = "语音会话已结束,请打开 OSGKeyboard 重新启动";
"keyboard.flow.startingSession" = "正在启动语音会话…"; "keyboard.flow.startingSession" = "正在启动语音会话…";
"keyboard.flow.transcribing" = "识别中…"; "keyboard.flow.transcribing" = "识别中…";
@@ -25,6 +25,14 @@ private final class FlowCaptureStreamRelay: @unchecked Sendable {
lock.withLock { self.continuation = continuation } lock.withLock { self.continuation = continuation }
} }
func replay(_ snapshots: [AudioBufferSnapshot]) {
lock.withLock {
for snapshot in snapshots {
continuation?.yield(snapshot)
}
}
}
func yield(_ snapshot: AudioBufferSnapshot) { func yield(_ snapshot: AudioBufferSnapshot) {
lock.withLock { continuation?.yield(snapshot) } lock.withLock { continuation?.yield(snapshot) }
} }
@@ -37,6 +45,34 @@ private final class FlowCaptureStreamRelay: @unchecked Sendable {
} }
} }
/// Rolling pre-roll while utterance gate is closed (~400 ms at typical tap rates).
private final class FlowPrerollStore: @unchecked Sendable {
private let lock = OSAllocatedUnfairLock()
private var snapshots: [AudioBufferSnapshot] = []
private let maxCount: Int
init(maxCount: Int = 6) {
self.maxCount = maxCount
}
func append(_ snapshot: AudioBufferSnapshot) {
lock.withLock {
snapshots.append(snapshot)
if snapshots.count > maxCount {
snapshots.removeFirst(snapshots.count - maxCount)
}
}
}
func drain() -> [AudioBufferSnapshot] {
lock.withLock {
let drained = snapshots
snapshots.removeAll()
return drained
}
}
}
/// Rolling bar levels updated from the audio tap; read on the main actor. /// Rolling bar levels updated from the audio tap; read on the main actor.
private final class FlowLevelStore: @unchecked Sendable { private final class FlowLevelStore: @unchecked Sendable {
private let lock = OSAllocatedUnfairLock() private let lock = OSAllocatedUnfairLock()
@@ -120,6 +156,7 @@ public final class FlowContinuousCapture {
private let audioEngine = AVAudioEngine() private let audioEngine = AVAudioEngine()
private let streamRelay = FlowCaptureStreamRelay() private let streamRelay = FlowCaptureStreamRelay()
private let prerollStore = FlowPrerollStore()
private let levelStore = FlowLevelStore(barCount: FlowCaptureConstants.levelBarCount) private let levelStore = FlowLevelStore(barCount: FlowCaptureConstants.levelBarCount)
private let isUtteranceActive = OSAllocatedUnfairLock(initialState: false) private let isUtteranceActive = OSAllocatedUnfairLock(initialState: false)
@@ -170,6 +207,7 @@ public final class FlowContinuousCapture {
if !didInstallTap { if !didInstallTap {
let utteranceFlag = isUtteranceActive let utteranceFlag = isUtteranceActive
let relay = streamRelay let relay = streamRelay
let preroll = prerollStore
let levels = levelStore let levels = levelStore
let tap = Self.makeAudioTapBlock( let tap = Self.makeAudioTapBlock(
converter: converter, converter: converter,
@@ -177,6 +215,7 @@ public final class FlowContinuousCapture {
hwFormat: hwFormat, hwFormat: hwFormat,
utteranceFlag: utteranceFlag, utteranceFlag: utteranceFlag,
levelStore: levels, levelStore: levels,
prerollStore: preroll,
streamRelay: relay streamRelay: relay
) )
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap) inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
@@ -213,9 +252,12 @@ public final class FlowContinuousCapture {
/// Begin forwarding downsampled buffers to ASR for one utterance. /// Begin forwarding downsampled buffers to ASR for one utterance.
public func beginUtterance() -> AsyncStream<AudioBufferSnapshot> { public func beginUtterance() -> AsyncStream<AudioBufferSnapshot> {
isUtteranceActive.withLock { $0 = true }
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream() let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
// Bind the consumer before opening the gate so early tap frames
// are not dropped on the floor.
streamRelay.bind(continuation) streamRelay.bind(continuation)
streamRelay.replay(prerollStore.drain())
isUtteranceActive.withLock { $0 = true }
return stream return stream
} }
@@ -241,13 +283,12 @@ public final class FlowContinuousCapture {
hwFormat: AVAudioFormat, hwFormat: AVAudioFormat,
utteranceFlag: OSAllocatedUnfairLock<Bool>, utteranceFlag: OSAllocatedUnfairLock<Bool>,
levelStore: FlowLevelStore, levelStore: FlowLevelStore,
prerollStore: FlowPrerollStore,
streamRelay: FlowCaptureStreamRelay streamRelay: FlowCaptureStreamRelay
) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void { ) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
return { buffer, _ in return { buffer, _ in
levelStore.update(from: buffer, barCount: FlowCaptureConstants.levelBarCount) levelStore.update(from: buffer, barCount: FlowCaptureConstants.levelBarCount)
guard utteranceFlag.withLock({ $0 }) else { return }
let outFrames = AVAudioFrameCount( let outFrames = AVAudioFrameCount(
Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate
) )
@@ -264,7 +305,12 @@ public final class FlowContinuousCapture {
let snapshot = AudioBufferSnapshot(buffer: outBuffer) let snapshot = AudioBufferSnapshot(buffer: outBuffer)
guard !snapshot.samples.isEmpty else { return } guard !snapshot.samples.isEmpty else { return }
streamRelay.yield(snapshot)
if utteranceFlag.withLock({ $0 }) {
streamRelay.yield(snapshot)
} else {
prerollStore.append(snapshot)
}
} }
} }
} }
@@ -82,6 +82,7 @@ public final class KeyboardState: ObservableObject {
public var endRecording: () -> Void = {} public var endRecording: () -> Void = {}
public var tapMic: () -> Void = {} public var tapMic: () -> Void = {}
public var openSettings: () -> Void = {} public var openSettings: () -> Void = {}
public var startFlowSession: () -> Void = {}
public var setMode: (InputMode) -> Void = { _ in } public var setMode: (InputMode) -> Void = { _ in }
public var setLocale: (String) -> Void = { _ in } public var setLocale: (String) -> Void = { _ in }
public var setEngineMode: (String) -> Void = { _ in } public var setEngineMode: (String) -> Void = { _ in }