rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: flow session reliability, keyboard UX, and audio pre-roll fix

Browse Source 
Renew voice sessions while the host app stays foreground, auto-start flow from the keyboard with a Start action, and prevent leading audio loss via pre-roll buffering and faster signal polling.
This commit is contained in:
Rocky
2026-06-20 00:57:16 +08:00
parent 38a36ed504
commit db5151ad9a
12 changed files with 196 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OSGKeyboard/OSGKeyboardApp.swift
+4
View File
@@ -35,6 +35,9 @@ struct OSGKeyboardApp: App {
}
}
.environmentObject(flowManager)
.onAppear {
flowManager.setAppForeground(scenePhase == .active)
}
.onOpenURL { url in
guard url.scheme == "osgkeyboard" else { return }
switch url.host {
@@ -56,6 +59,7 @@ struct OSGKeyboardApp: App {
if done { flowManager.autoStartIfNeeded() }
}
.onChange(of: scenePhase) { _, phase in
flowManager.setAppForeground(phase == .active)
guard phase == .active, AppGroup.isAvailable, config.hasCompletedOnboarding else { return }
if flowManager.isActive {
flowManager.extendSession()
OSGKeyboard/Services/FlowSessionManager.swift
+22 -1
View File
@@ -35,6 +35,8 @@ final class FlowSessionManager: ObservableObject {
private var asrTask: Task<Void, Never>?
private var currentPartial = ""
private var lastFinal = ""
/// True while the host app scene is `.active` — drives foreground renewal.
private var isAppForeground = false
init() {
Task { @MainActor [weak self] in
@@ -169,6 +171,24 @@ final class FlowSessionManager: ObservableObject {
scheduleExpiry(after: duration)
}
/// Called from `OSGKeyboardApp` when `scenePhase` changes.
func setAppForeground(_ foreground: Bool) {
isAppForeground = foreground
if foreground, isActive {
renewSessionIfNeededWhileForeground()
}
}
/// Extend the session before it expires while the host app stays in foreground.
private func renewSessionIfNeededWhileForeground() {
guard isActive, isAppForeground else { return }
guard let remaining = FlowSessionBridge.remainingSessionDuration() else { return }
let threshold = FlowSessionKeys.defaultSessionDuration * 0.25
guard remaining < threshold else { return }
extendSession()
debug("Flow session renewed in foreground (\(Int(threshold))s threshold)")
}
// MARK: - Session start
private func startSessionAsync(duration: TimeInterval) async {
@@ -217,7 +237,7 @@ final class FlowSessionManager: ObservableObject {
pollingTask = Task { @MainActor [weak self] in
while !Task.isCancelled {
self?.handleKeyboardSignal()
try? await Task.sleep(nanoseconds: 100_000_000)
try? await Task.sleep(nanoseconds: 50_000_000)
}
}
}
@@ -424,6 +444,7 @@ final class FlowSessionManager: ObservableObject {
heartbeatTask = Task { @MainActor [weak self] in
while !Task.isCancelled {
FlowSessionBridge.writeHeartbeat()
self?.renewSessionIfNeededWhileForeground()
try? await Task.sleep(nanoseconds: 1_000_000_000)
guard self?.isActive == true else { break }
}
OSGKeyboard/Views/HomeView.swift
+3 -22
View File
@@ -42,8 +42,8 @@ struct HomeView: View {
engineStatusLine
.padding(.horizontal, Spacing.lg)
.padding(.top, Spacing.md)
.padding(.bottom, Spacing.xs)
.padding(.top, Spacing.xl)
.padding(.bottom, Spacing.sm)
}
.frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top)
}
@@ -81,7 +81,7 @@ struct HomeView: View {
Image("osglogo")
.resizable()
.scaledToFit()
.frame(width: 120, height: 34)
.frame(width: 144, height: 41)
.accessibilityHidden(true)
statusCapsule
@@ -92,13 +92,6 @@ struct HomeView: View {
private var statusCapsule: some View {
HStack(spacing: Spacing.sm) {
Text(statusLine)
.font(TypeStyle.status)
.foregroundStyle(palette.textSecondary)
.fixedSize()
capsuleDivider
flowCapsuleSegment
if flowManager.isActive {
@@ -125,12 +118,6 @@ struct HomeView: View {
.animation(Motion.soft, value: flowManager.isActive)
}
private var capsuleDivider: some View {
Circle()
.fill(palette.dividerStrong)
.frame(width: 3, height: 3)
}
@ViewBuilder
private var flowCapsuleSegment: some View {
HStack(spacing: Spacing.xs) {
@@ -165,12 +152,6 @@ struct HomeView: View {
: palette.surface.opacity(0.88)
}
private var statusLine: String {
config.isConfigured
? NSLocalizedString("home.status.ready", comment: "")
: NSLocalizedString("home.status.setupIncomplete", comment: "")
}
// MARK: - Flow extras (warnings / hint)
@ViewBuilder
OSGKeyboard/Views/OnboardingView.swift
+17 -7
View File
@@ -184,6 +184,11 @@ private struct OnboardingHeroIcon: View {
}
}
private enum OnboardingLayoutMetrics {
/// Matches welcome page: logo → tagline, and subtitle → next block.
static let heroTextGap: CGFloat = Spacing.hero
}
// MARK: - Welcome
private struct WelcomePage: View {
@@ -209,11 +214,11 @@ private struct WelcomePage: View {
}
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.xl)
.padding(.top, Spacing.hero)
.padding(.top, OnboardingLayoutMetrics.heroTextGap)
PrivacyFootnote()
.padding(.horizontal, Spacing.xl)
.padding(.top, Spacing.hero)
.padding(.top, OnboardingLayoutMetrics.heroTextGap)
if let url = LegalLinks.privacyPolicyURL {
Link(destination: url) {
@@ -388,12 +393,12 @@ private struct PermissionPageLayout: View {
var deniedHint: LocalizedStringKey? = nil
var body: some View {
VStack(spacing: Spacing.xl) {
VStack(spacing: 0) {
Spacer()
OnboardingHeroIcon(systemName: icon, circleSize: 96, iconSize: 40)
VStack(spacing: Spacing.sm) {
VStack(spacing: Spacing.xs) {
Text(title)
.font(TypeStyle.title2)
.foregroundStyle(palette.textPrimary)
@@ -403,6 +408,7 @@ private struct PermissionPageLayout: View {
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.xl)
}
.padding(.top, OnboardingLayoutMetrics.heroTextGap)
HStack(spacing: 6) {
Circle().fill(statusColor).frame(width: 8, height: 8)
@@ -410,6 +416,7 @@ private struct PermissionPageLayout: View {
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
}
.padding(.top, OnboardingLayoutMetrics.heroTextGap)
if let deniedHint {
Text(deniedHint)
@@ -417,6 +424,7 @@ private struct PermissionPageLayout: View {
.foregroundStyle(palette.warning)
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.xl)
.padding(.top, Spacing.md)
}
VStack(spacing: Spacing.sm) {
@@ -436,6 +444,7 @@ private struct PermissionPageLayout: View {
}
}
.padding(.horizontal, Spacing.lg)
.padding(.top, Spacing.xl)
Spacer()
}
@@ -448,12 +457,12 @@ private struct EnableKeyboardPage: View {
@Environment(\.themePalette) private var palette: ThemePalette
var body: some View {
VStack(spacing: Spacing.xl) {
VStack(spacing: 0) {
Spacer()
OnboardingHeroIcon(systemName: "keyboard.fill", circleSize: 96, iconSize: 40)
VStack(spacing: Spacing.sm) {
VStack(spacing: Spacing.xs) {
Text("onboarding.enable.title")
.font(TypeStyle.title2)
.foregroundStyle(palette.textPrimary)
@@ -463,6 +472,7 @@ private struct EnableKeyboardPage: View {
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.lg)
}
.padding(.top, OnboardingLayoutMetrics.heroTextGap)
VStack(alignment: .leading, spacing: Spacing.lg) {
step(num: 1, text: NSLocalizedString("onboarding.enable.step1", comment: ""))
@@ -472,7 +482,7 @@ private struct EnableKeyboardPage: View {
}
.frame(maxWidth: .infinity, alignment: .leading)
.padding(.horizontal, Spacing.xl)
.padding(.top, Spacing.sm)
.padding(.top, OnboardingLayoutMetrics.heroTextGap)
Button {
AppPermissions.openSystemSettings()
OSGKeyboard/en.lproj/Localizable.strings
+8 -8
View File
@@ -7,11 +7,11 @@
"onboarding.welcome.subtitle" = "Your open-source voice keyboard";
"onboarding.enable.fullAccessNote" = "Full Access lets the keyboard reach the microphone and your LLM API. We never log what you type.";
"onboarding.permission.mic.title" = "Microphone access";
"onboarding.permission.mic.body" = "Required for voice input. Audio is processed on your device for transcription.";
"onboarding.permission.mic.body" = "Voice input requires your permission to use the microphone.";
"onboarding.permission.mic.allow" = "Allow microphone";
"onboarding.permission.mic.deniedHint" = "Microphone was denied. Open Settings to enable it, or tap Next to continue.";
"onboarding.permission.speech.title" = "Speech recognition";
"onboarding.permission.speech.body" = "On-device speech recognition turns your voice into text. Audio never leaves your device.";
"onboarding.permission.speech.body" = "On-device speech recognition turns your voice into text.";
"onboarding.permission.speech.allow" = "Allow speech recognition";
"onboarding.permission.speech.deniedHint" = "Speech recognition was denied. Open Settings to enable it, or tap Next to continue.";
"onboarding.permission.openSettings" = "Open Settings";
@@ -43,12 +43,12 @@
"common.newline" = "Return";
/* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "Audio stays on device";
"privacy.audio.body" = "Transcribed locally with Apple’s speech engine.";
"privacy.network.title" = "Only the polished text is sent";
"privacy.network.body" = "Sent to your chosen LLM to add structure & punctuation.";
"privacy.audio.title" = "On-device transcription";
"privacy.audio.body" = "Powered by the on-device iOS speech engine.";
"privacy.network.title" = "Automatic text polish";
"privacy.network.body" = "AI formats and polishes your text automatically.";
"privacy.universal.title" = "Works everywhere";
"privacy.universal.body" = "WeChat, Notes, Mail, ChatGPT, Claude, Cursor — anywhere a keyboard appears.";
"privacy.universal.body" = "Chat, email, vibe coding — just speak.";
/* Home */
"home.status.ready" = "Ready";
@@ -184,7 +184,7 @@
/* Home · Flow session */
"home.flow.active" = "Voice session active";
"home.flow.label" = "Voice session";
"home.flow.label" = "Ready";
"home.flow.inactive" = "Voice session inactive";
"home.flow.hint" = "Voice session starts automatically. Switch to any app and tap the keyboard mic to dictate.";
"home.flow.starting" = "Starting voice session…";
OSGKeyboard/zh-Hans.lproj/Localizable.strings
+8 -8
View File
@@ -7,11 +7,11 @@
"onboarding.welcome.subtitle" = "你的开源语音输入法";
"onboarding.enable.fullAccessNote" = "完全访问用于麦克风与 API 配置读取。我们不会记录或上传你的击键内容。";
"onboarding.permission.mic.title" = "麦克风权限";
"onboarding.permission.mic.body" = "语音输入需要麦克风。音频在设备端转录，不会上传原始录音。";
"onboarding.permission.mic.body" = "语音输入需要您授予麦克风权限";
"onboarding.permission.mic.allow" = "允许麦克风";
"onboarding.permission.mic.deniedHint" = "麦克风被拒绝。可前往设置开启，或点「下一步」继续。";
"onboarding.permission.speech.title" = "语音识别权限";
"onboarding.permission.speech.body" = "端侧语音识别将语音转为文字。音频不会离开你的设备。";
"onboarding.permission.speech.body" = "端侧语音识别将语音转为文字。";
"onboarding.permission.speech.allow" = "允许语音识别";
"onboarding.permission.speech.deniedHint" = "语音识别被拒绝。可前往设置开启，或点「下一步」继续。";
"onboarding.permission.openSettings" = "打开设置";
@@ -43,12 +43,12 @@
"common.newline" = "换行";
/* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "音频不出本机";
"privacy.audio.body" = "由 Apple 端侧引擎转录。";
"privacy.network.title" = "仅发送润色后文字";
"privacy.network.body" = "仅向所选 LLM 发送润色后文字,用于整理结构和标点。";
"privacy.audio.title" = "支持本地转写";
"privacy.audio.body" = "支持 iOS 本地引擎转录";
"privacy.network.title" = "文字自动润色";
"privacy.network.body" = "通过 AI 自动对文字进行排版润色";
"privacy.universal.title" = "处处可用";
"privacy.universal.body" = "微信、备忘录、邮件、ChatGPT、Claude、Cursor — 任何键盘出现的地方。";
"privacy.universal.body" = "聊天、邮件、Vibe Coding 动嘴就行";
/* Home */
"home.status.ready" = "就绪";
@@ -183,7 +183,7 @@
/* Home · Flow session */
"home.flow.active" = "语音会话进行中";
"home.flow.label" = "语音会话";
"home.flow.label" = "就绪";
"home.flow.inactive" = "语音会话未启动";
"home.flow.hint" = "语音会话会自动启动。切到任意 App，点按键盘麦克风即可说话。";
"home.flow.starting" = "正在启动语音会话…";
OSGKeyboardExt/KeyboardViewController.swift
+22
View File
@@ -66,6 +66,8 @@ public final class KeyboardViewController: UIInputViewController {
private var flowSessionMonitorTask: Task<Void, Never>?
private var flowSessionDarwinObserver: FlowSessionDarwinObserver?
private var isAwaitingFlowResult = false
private var lastFlowAutoStartAttempt: TimeInterval = 0
private static let flowAutoStartCooldown: TimeInterval = 20
// MARK: - Lifecycle
@@ -121,6 +123,7 @@ public final class KeyboardViewController: UIInputViewController {
state.endRecording = { [weak self] in self?.pressEnded() }
state.tapMic = { [weak self] in self?.toggleRecording() }
state.openSettings = { [weak self] in self?.openHostApp() }
state.startFlowSession = { [weak self] in self?.beginFlowStart() }
state.setMode = { [weak self] m in self?.persistMode(m) }
state.setLocale = { [weak self] l in self?.persistLocale(l) }
state.setEngineMode = { [weak self] m in self?.persistEngineMode(m) }
@@ -199,6 +202,24 @@ public final class KeyboardViewController: UIInputViewController {
}
}
wasFlowSessionActive = active
if !active {
maybeAutoStartFlowSession()
}
}
/// When the host session is down, proactively jump to the app to start it.
private func maybeAutoStartFlowSession() {
guard !FlowSessionBridge.isSessionActive() else { return }
guard !isPendingFlowStart, !isFlowRecording, !isAwaitingFlowResult else { return }
guard state.mode != .off else { return }
guard hasFullAccess, AppGroup.isAvailable else { return }
guard case .idle = state.phase else { return }
let now = Date().timeIntervalSince1970
guard now - lastFlowAutoStartAttempt >= Self.flowAutoStartCooldown else { return }
lastFlowAutoStartAttempt = now
beginFlowStart()
}
private func showFlowSessionExpiredHint() {
@@ -303,6 +324,7 @@ public final class KeyboardViewController: UIInputViewController {
}
private func beginFlowStart() {
guard !isPendingFlowStart else { return }
isPendingFlowStart = true
isFlowRecording = false
flowStartDeadline = Date().timeIntervalSince1970 + FlowWatchdog.startTimeout
OSGKeyboardExt/Views/KeyboardRootView.swift
+42 -12
View File
@@ -19,7 +19,13 @@ import SwiftUI
import OSGKeyboardShared
private enum KeyboardLayoutMetrics {
static let sideActionButtonSize: CGFloat = 44
static let sideActionButtonSize: CGFloat = 53
static let sideActionIconSize: CGFloat = 19
static let sideSpaceBarWidth: CGFloat = 19
static let micFlankMinSpacing: CGFloat = 36
static let sideActionStackSpacing: CGFloat = 16
/// Outer inset for delete / return·space from screen edges (8 pt → 24 pt, +200%).
static let sideActionHorizontalInset: CGFloat = Spacing.xs * 3
}
public struct KeyboardRootView: View {
@@ -36,7 +42,6 @@ public struct KeyboardRootView: View {
/// it up.
static let totalHeight: CGFloat = 280
private static let topBarHeight: CGFloat = 38
private static let sideActionStackSpacing: CGFloat = 10
private var palette: ThemePalette {
colorScheme == .dark ? Palette.dark : Palette.light
@@ -52,14 +57,16 @@ public struct KeyboardRootView: View {
}
.padding(.top, 4)
.padding(.bottom, 6)
// Let the system UI chrome show through by drawing no background
// of our own.
.background(Color.clear)
.background(keyboardBackground)
.frame(height: Self.totalHeight)
// Feed the resolved palette to all nested chips/buttons.
.environment(\.themePalette, palette)
}
private var keyboardBackground: Color {
colorScheme == .dark ? palette.background : Color.clear
}
// MARK: - Top bar
private var topBar: some View {
@@ -99,7 +106,8 @@ public struct KeyboardRootView: View {
phase: state.phase,
transcript: state.lastTranscript,
flowSessionActive: state.flowSessionActive,
openSettings: state.openSettings
openSettings: state.openSettings,
startFlowSession: state.startFlowSession
)
.frame(height: 22)
@@ -117,11 +125,13 @@ public struct KeyboardRootView: View {
/// HStack vertical alignment keeps delete, mic centre, and the gap
/// between return/space on one horizontal axis.
private var micActionRow: some View {
HStack(alignment: .center, spacing: 20) {
HStack(alignment: .center, spacing: 0) {
CircularToolbarButton(systemName: "delete.left", label: "delete") {
state.deleteBackward()
}
Spacer(minLength: KeyboardLayoutMetrics.micFlankMinSpacing)
RecordButton(
phase: buttonPhase,
level: state.level,
@@ -130,7 +140,9 @@ public struct KeyboardRootView: View {
)
.frame(width: 132, height: 132)
VStack(spacing: Self.sideActionStackSpacing) {
Spacer(minLength: KeyboardLayoutMetrics.micFlankMinSpacing)
VStack(spacing: KeyboardLayoutMetrics.sideActionStackSpacing) {
CircularToolbarButton(systemName: "return", label: "newline") {
state.insertNewline()
}
@@ -139,6 +151,7 @@ public struct KeyboardRootView: View {
}
}
}
.padding(.horizontal, KeyboardLayoutMetrics.sideActionHorizontalInset)
.frame(maxWidth: .infinity)
}
@@ -191,6 +204,7 @@ private struct TranscriptLine: View {
let transcript: String
let flowSessionActive: Bool
let openSettings: () -> Void
let startFlowSession: () -> Void
var body: some View {
ZStack {
@@ -201,9 +215,18 @@ private struct TranscriptLine: View {
.font(TypeStyle.caption)
.foregroundStyle(palette.textTertiary)
} else {
HStack(spacing: 6) {
ExtL10n.text("keyboard.flow.sessionInactive")
.font(TypeStyle.caption)
.foregroundStyle(palette.textTertiary)
Button(action: startFlowSession) {
ExtL10n.text("keyboard.flow.start")
.font(TypeStyle.caption)
.foregroundStyle(palette.accent)
}
.buttonStyle(.plain)
.accessibilityHint(ExtL10n.text("keyboard.flow.startA11y"))
}
}
case .requestingPermissions:
HStack(spacing: 6) {
@@ -267,6 +290,7 @@ private struct TranscriptLine: View {
// MARK: - Circular toolbar button
private struct CircularToolbarButton: View {
@Environment(\.colorScheme) private var colorScheme
@Environment(\.themePalette) private var palette: ThemePalette
let systemName: String?
@@ -294,20 +318,26 @@ private struct CircularToolbarButton: View {
if spaceStyle {
Capsule()
.fill(palette.textPrimary)
.frame(width: 16, height: 3)
.frame(width: KeyboardLayoutMetrics.sideSpaceBarWidth, height: 3)
} else if let systemName {
Image(systemName: systemName)
.font(.system(size: 16, weight: .medium))
.font(.system(size: KeyboardLayoutMetrics.sideActionIconSize, weight: .medium))
.foregroundStyle(palette.textPrimary)
}
}
.frame(width: KeyboardLayoutMetrics.sideActionButtonSize, height: KeyboardLayoutMetrics.sideActionButtonSize)
.background(palette.surfaceElevated, in: Circle())
.overlay(Circle().stroke(palette.divider, lineWidth: 0.5))
.background(sideButtonFill, in: Circle())
.overlay(Circle().stroke(palette.dividerStrong, lineWidth: 0.5))
}
.buttonStyle(.plain)
.accessibilityLabel(Text(label))
}
private var sideButtonFill: Color {
colorScheme == .dark
? Color(red: 0.20, green: 0.20, blue: 0.22)
: palette.surfaceElevated
}
}
// MARK: - Status badge
OSGKeyboardExt/en.lproj/Keyboard.strings
+8 -6
View File
@@ -27,12 +27,12 @@
"common.newline" = "Return";
/* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "Audio stays on device";
"privacy.audio.body" = "Transcribed locally with Apple’s speech engine.";
"privacy.network.title" = "Only the polished text is sent";
"privacy.network.body" = "Sent to your chosen LLM to add structure & punctuation.";
"privacy.audio.title" = "On-device transcription";
"privacy.audio.body" = "Powered by the on-device iOS speech engine.";
"privacy.network.title" = "Automatic text polish";
"privacy.network.body" = "AI formats and polishes your text automatically.";
"privacy.universal.title" = "Works everywhere";
"privacy.universal.body" = "WeChat, Notes, Mail, ChatGPT, Claude, Cursor — anywhere a keyboard appears.";
"privacy.universal.body" = "Chat, email, vibe coding — just speak.";
/* Home */
"home.status.ready" = "Ready";
@@ -127,7 +127,9 @@
"keyboard.tapToTalkA11y" = "Tap to talk";
/* Flow session (keyboard) */
"keyboard.flow.sessionInactive" = "Open OSGKeyboard to start voice session";
"keyboard.flow.sessionInactive" = "Voice session off";
"keyboard.flow.start" = "Start";
"keyboard.flow.startA11y" = "Start voice session";
"keyboard.flow.sessionExpired" = "Voice session ended. Open OSGKeyboard to restart.";
"keyboard.flow.startingSession" = "Starting voice session…";
"keyboard.flow.transcribing" = "Transcribing…";
OSGKeyboardExt/zh-Hans.lproj/Keyboard.strings
+8 -6
View File
@@ -27,12 +27,12 @@
"common.newline" = "换行";
/* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "音频不出本机";
"privacy.audio.body" = "由 Apple 端侧引擎转录。";
"privacy.network.title" = "仅发送润色后文字";
"privacy.network.body" = "仅向所选 LLM 发送润色后文字,用于整理结构和标点。";
"privacy.audio.title" = "支持本地转写";
"privacy.audio.body" = "支持 iOS 本地引擎转录";
"privacy.network.title" = "文字自动润色";
"privacy.network.body" = "通过 AI 自动对文字进行排版润色";
"privacy.universal.title" = "处处可用";
"privacy.universal.body" = "微信、备忘录、邮件、ChatGPT、Claude、Cursor — 任何键盘出现的地方。";
"privacy.universal.body" = "聊天、邮件、Vibe Coding 动嘴就行";
/* Home */
"home.status.ready" = "就绪";
@@ -127,7 +127,9 @@
"keyboard.tapToTalkA11y" = "点按说话";
/* Flow session (keyboard) */
"keyboard.flow.sessionInactive" = "请打开 OSGKeyboard 启动语音会话";
"keyboard.flow.sessionInactive" = "语音会话未启动";
"keyboard.flow.start" = "启动";
"keyboard.flow.startA11y" = "启动语音会话";
"keyboard.flow.sessionExpired" = "语音会话已结束，请打开 OSGKeyboard 重新启动";
"keyboard.flow.startingSession" = "正在启动语音会话…";
"keyboard.flow.transcribing" = "识别中…";
OSGKeyboardShared/Services/FlowContinuousCapture.swift
+49 -3
View File
@@ -25,6 +25,14 @@ private final class FlowCaptureStreamRelay: @unchecked Sendable {
lock.withLock { self.continuation = continuation }
}
func replay(_ snapshots: [AudioBufferSnapshot]) {
lock.withLock {
for snapshot in snapshots {
continuation?.yield(snapshot)
}
}
}
func yield(_ snapshot: AudioBufferSnapshot) {
lock.withLock { continuation?.yield(snapshot) }
}
@@ -37,6 +45,34 @@ private final class FlowCaptureStreamRelay: @unchecked Sendable {
}
}
/// Rolling pre-roll while utterance gate is closed (~400 ms at typical tap rates).
private final class FlowPrerollStore: @unchecked Sendable {
private let lock = OSAllocatedUnfairLock()
private var snapshots: [AudioBufferSnapshot] = []
private let maxCount: Int
init(maxCount: Int = 6) {
self.maxCount = maxCount
}
func append(_ snapshot: AudioBufferSnapshot) {
lock.withLock {
snapshots.append(snapshot)
if snapshots.count > maxCount {
snapshots.removeFirst(snapshots.count - maxCount)
}
}
}
func drain() -> [AudioBufferSnapshot] {
lock.withLock {
let drained = snapshots
snapshots.removeAll()
return drained
}
}
}
/// Rolling bar levels updated from the audio tap; read on the main actor.
private final class FlowLevelStore: @unchecked Sendable {
private let lock = OSAllocatedUnfairLock()
@@ -120,6 +156,7 @@ public final class FlowContinuousCapture {
private let audioEngine = AVAudioEngine()
private let streamRelay = FlowCaptureStreamRelay()
private let prerollStore = FlowPrerollStore()
private let levelStore = FlowLevelStore(barCount: FlowCaptureConstants.levelBarCount)
private let isUtteranceActive = OSAllocatedUnfairLock(initialState: false)
@@ -170,6 +207,7 @@ public final class FlowContinuousCapture {
if !didInstallTap {
let utteranceFlag = isUtteranceActive
let relay = streamRelay
let preroll = prerollStore
let levels = levelStore
let tap = Self.makeAudioTapBlock(
converter: converter,
@@ -177,6 +215,7 @@ public final class FlowContinuousCapture {
hwFormat: hwFormat,
utteranceFlag: utteranceFlag,
levelStore: levels,
prerollStore: preroll,
streamRelay: relay
)
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
@@ -213,9 +252,12 @@ public final class FlowContinuousCapture {
/// Begin forwarding downsampled buffers to ASR for one utterance.
public func beginUtterance() -> AsyncStream<AudioBufferSnapshot> {
isUtteranceActive.withLock { $0 = true }
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
// Bind the consumer before opening the gate so early tap frames
// are not dropped on the floor.
streamRelay.bind(continuation)
streamRelay.replay(prerollStore.drain())
isUtteranceActive.withLock { $0 = true }
return stream
}
@@ -241,13 +283,12 @@ public final class FlowContinuousCapture {
hwFormat: AVAudioFormat,
utteranceFlag: OSAllocatedUnfairLock<Bool>,
levelStore: FlowLevelStore,
prerollStore: FlowPrerollStore,
streamRelay: FlowCaptureStreamRelay
) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
return { buffer, _ in
levelStore.update(from: buffer, barCount: FlowCaptureConstants.levelBarCount)
guard utteranceFlag.withLock({ $0 }) else { return }
let outFrames = AVAudioFrameCount(
Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate
)
@@ -264,7 +305,12 @@ public final class FlowContinuousCapture {
let snapshot = AudioBufferSnapshot(buffer: outBuffer)
guard !snapshot.samples.isEmpty else { return }
if utteranceFlag.withLock({ $0 }) {
streamRelay.yield(snapshot)
} else {
prerollStore.append(snapshot)
}
}
}
}
OSGKeyboardShared/Services/KeyboardState.swift
+1
View File
@@ -82,6 +82,7 @@ public final class KeyboardState: ObservableObject {
public var endRecording: () -> Void = {}
public var tapMic: () -> Void = {}
public var openSettings: () -> Void = {}
public var startFlowSession: () -> Void = {}
public var setMode: (InputMode) -> Void = { _ in }
public var setLocale: (String) -> Void = { _ in }
public var setEngineMode: (String) -> Void = { _ in }