feat(flow): harden host handoff, session UX, and Live Activity brand

- fix(keyboard): drop unusable extensionContext.open; report the real
  result of UIApplication.open and cancel the start watchdog immediately
  on failure with clear "open OSGKeyboard" guidance instead of a 30s spin
- feat(capture): recover from audio route changes (AirPods/wired headset
  plug-unplug) and phone-call interruptions by rebuilding the engine/tap
  against the live hardware format
- feat(home): add a "Start" button in the Home footer when a session is
  inactive and permissions are granted (no-jump manual restart)
- feat(hostapps): expand the return whitelist to 46 apps (Signal, Zoom,
  Instagram, X, Douyin, Zhihu, Bilibili, Things, Firefox, ...) with
  en/zh-Hans names and synced LSApplicationQueriesSchemes
- feat(island): use a "ready" checkmark instead of a mic glyph while idle
  and set a staleDate so an orphaned Live Activity fades after a force-quit
- feat(coldstart): use the template OSG brand mark instead of a system
  waveform glyph in the cold-start overlay
- perf(keyboard): rely on Darwin notifications as the primary session
  signal and drop the fallback poll from 1s to 3s
- fix(keyboard): remove the redundant "Start" text button next to the
  inactive-session hint on the keyboard
- docs: correct the per-take cap to 210s (3.5 min) and document that
  force-quitting no longer resurrects the session (tracker + READMEs)
This commit is contained in:
Rocky
2026-07-06 17:56:10 +08:00
parent 6a01b6cf71
commit 9ef0196d52
24 changed files with 560 additions and 197 deletions
@@ -0,0 +1,16 @@
{
"images" : [
{
"filename" : "OSGLogo.svg",
"idiom" : "universal"
}
],
"info" : {
"author" : "xcode",
"version" : 1
},
"properties" : {
"preserves-vector-representation" : true,
"template-rendering-intent" : "template"
}
}
@@ -0,0 +1,7 @@
<svg width="1024" height="1024" viewBox="0 0 1024 1024" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M627.511 512.419C627.511 448.783 576.127 397.197 512.741 397.197C449.355 397.197 397.971 448.783 397.971 512.419C397.971 576.054 449.355 627.641 512.741 627.641V637.837C443.746 637.837 387.814 581.686 387.814 512.419C387.814 443.152 443.746 387 512.741 387C581.736 387 637.668 443.152 637.668 512.419C637.668 581.686 581.736 637.837 512.741 637.837V627.641C576.127 627.641 627.511 576.054 627.511 512.419Z" fill="white"/>
<path d="M56.2632 512.419C56.2632 581.686 112.195 637.837 181.19 637.837C250.185 637.837 306.117 581.686 306.117 512.419C306.117 443.152 250.185 387 181.19 387C112.195 387 56.2632 443.152 56.2632 512.419Z" fill="white"/>
<path d="M399.618 562.094L399.618 552.257L512.74 552.257L512.74 562.094L399.618 562.094Z" fill="white"/>
<path d="M512.741 483.398V473.561H625.864V483.398H512.741Z" fill="white"/>
<path d="M843.308 387.981C910.987 387.981 966.093 441.8 968.171 508.975H843.308V518.812H968.096C965.014 585.066 910.324 637.836 843.308 637.836C774.313 637.836 718.381 581.904 718.381 512.909C718.381 443.914 774.312 387.981 843.308 387.981ZM968.234 518.812H968.096C968.187 516.855 968.234 514.888 968.234 512.909C968.234 511.593 968.211 510.281 968.171 508.975H968.234V518.812Z" fill="white"/>
</svg>

After

Width:  |  Height:  |  Size: 1.3 KiB

+32 -8
View File
@@ -34,6 +34,12 @@
</array>
</dict>
</array>
<key>CFBundleVersion</key>
<string>$(CURRENT_PROJECT_VERSION)</string>
<key>ITSAppUsesNonExemptEncryption</key>
<false/>
<key>LSApplicationCategoryType</key>
<string>public.app-category.utilities</string>
<key>LSApplicationQueriesSchemes</key>
<array>
<string>weixin</string>
@@ -57,13 +63,31 @@
<string>googlechrome</string>
<string>sinaweibo</string>
<string>xhsdiscover</string>
<string>sgnl</string>
<string>kakaotalk</string>
<string>viber</string>
<string>zalo</string>
<string>skype</string>
<string>zoomus</string>
<string>things</string>
<string>todoist</string>
<string>evernote</string>
<string>onenote</string>
<string>readdle-spark</string>
<string>firefox</string>
<string>microsoft-edge</string>
<string>fb</string>
<string>instagram</string>
<string>twitter</string>
<string>barcelona</string>
<string>snapchat</string>
<string>reddit</string>
<string>pinterest</string>
<string>zhihu</string>
<string>bilibili</string>
<string>snssdk1128</string>
<string>tiktok</string>
</array>
<key>CFBundleVersion</key>
<string>$(CURRENT_PROJECT_VERSION)</string>
<key>ITSAppUsesNonExemptEncryption</key>
<false/>
<key>LSApplicationCategoryType</key>
<string>public.app-category.utilities</string>
<key>NSAppTransportSecurity</key>
<dict>
<key>NSAllowsArbitraryLoads</key>
@@ -73,6 +97,8 @@
<string>OSGKeyboard uses the microphone for voice dictation and keeps a background audio session active while a voice session is running.</string>
<key>NSSpeechRecognitionUsageDescription</key>
<string>OSGKeyboard uses on-device speech recognition to transcribe your voice. Audio is processed on your device and is not uploaded for transcription.</string>
<key>NSSupportsLiveActivities</key>
<true/>
<key>UIApplicationSceneManifest</key>
<dict>
<key>UIApplicationSupportsMultipleScenes</key>
@@ -82,8 +108,6 @@
<array>
<string>audio</string>
</array>
<key>NSSupportsLiveActivities</key>
<true/>
<key>UILaunchScreen</key>
<dict>
<key>UIColorName</key>
@@ -8,14 +8,28 @@ import ActivityKit
import Foundation
import OSGKeyboardShared
@MainActor
enum FlowLiveActivityController {
private static var currentActivity: Activity<FlowActivityAttributes>?
nonisolated(unsafe) private static var currentActivity: Activity<FlowActivityAttributes>?
/// If the host app is force-quit its `endSession()` never runs, orphaning
/// the Live Activity. A `staleDate` lets the system grey it out and become
/// willing to reclaim it without our process refreshed on every update
/// so a genuinely active, in-use session never looks stale.
private static let staleWindow: TimeInterval = 60 * 60
private static func freshContent(
phase: FlowActivityAttributes.ContentState.Phase
) -> ActivityContent<FlowActivityAttributes.ContentState> {
ActivityContent(
state: FlowActivityAttributes.ContentState(phase: phase),
staleDate: Date().addingTimeInterval(staleWindow)
)
}
/// Begin showing OSGKeyboard in the Dynamic Island for an active Flow session.
static func startSession() {
guard ActivityAuthorizationInfo().areActivitiesEnabled else {
debug("Live Activity disabled in Settings")
FlowDiagnostics.log("Live Activity disabled in Settings")
return
}
@@ -26,25 +40,21 @@ enum FlowLiveActivityController {
return
}
let state = FlowActivityAttributes.ContentState(phase: .idle)
let content = ActivityContent(state: state, staleDate: nil)
do {
currentActivity = try Activity.request(
attributes: FlowActivityAttributes(),
content: content,
content: freshContent(phase: .idle),
pushType: nil
)
debug("Live Activity started")
FlowDiagnostics.log("Live Activity started")
} catch {
debug("Live Activity start failed: \(error.localizedDescription)")
FlowDiagnostics.log("Live Activity start failed: \(error.localizedDescription)")
}
}
static func update(phase: FlowActivityAttributes.ContentState.Phase) {
guard let activity = currentActivity else { return }
let state = FlowActivityAttributes.ContentState(phase: phase)
let content = ActivityContent(state: state, staleDate: nil)
let content = freshContent(phase: phase)
Task {
await activity.update(content)
}
@@ -60,17 +70,19 @@ enum FlowLiveActivityController {
currentActivity = nil
Task {
await activity.end(nil, dismissalPolicy: .immediate)
debug("Live Activity ended")
FlowDiagnostics.log("Live Activity ended")
}
}
/// Host relaunch can leave orphan activities; clear them before starting anew.
private static func endStaleActivities() {
for activity in Activity<FlowActivityAttributes>.activities {
Task {
let staleActivities = Activity<FlowActivityAttributes>.activities
currentActivity = nil
guard !staleActivities.isEmpty else { return }
Task {
for activity in staleActivities {
await activity.end(nil, dismissalPolicy: .immediate)
}
}
currentActivity = nil
}
}
+12 -71
View File
@@ -67,9 +67,10 @@ final class FlowSessionManager: ObservableObject {
private var isColdStartHandoff = false
init() {
Task { @MainActor [weak self] in
await self?.bootstrapFromStorageIfNeeded()
}
// Sessions are (re)started explicitly on app foreground via
// `activateOnForeground()`. We deliberately do NOT silently reattach a
// stored session here after a force-quit that would resurrect capture
// (and keep a stale Live Activity alive) without the user re-opening.
}
// MARK: - Public
@@ -95,6 +96,8 @@ final class FlowSessionManager: ObservableObject {
return
}
guard !isStarting else { return }
startTask?.cancel()
startTask = Task { @MainActor [weak self] in
await self?.startSessionAsync(duration: duration)
@@ -102,23 +105,15 @@ final class FlowSessionManager: ObservableObject {
}
}
/// Restores an in-flight session after relaunch; does not auto-start a new one.
func restoreSessionIfNeeded() {
/// Auto-start (or renew) the Flow session on every app foreground when
/// permissions allow the "always auto-open, no off switch" policy. Also
/// clears any orphaned Live Activity a previously force-quit process left
/// behind (its `endSession()` could not run at kill time).
func activateOnForeground() {
guard AppGroup.isAvailable else { return }
guard !isActive, !isStarting else { return }
let markedActive = AppGroup.defaults.bool(forKey: FlowSessionKeys.flowSessionActive)
if markedActive, FlowSessionBridge.remainingSessionDuration() != nil {
Task { await bootstrapFromStorageIfNeeded() }
}
}
/// One-shot warmup after onboarding when permissions are already granted.
func warmupAfterOnboardingIfNeeded() {
guard AppGroup.isAvailable else { return }
guard !isActive, !isStarting else { return }
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
FlowLiveActivityController.endSession()
return
}
startSession()
@@ -134,60 +129,6 @@ final class FlowSessionManager: ObservableObject {
dismissColdStartOverlay()
}
/// Reattach capture when the host app was killed but the session has not expired.
func bootstrapFromStorageIfNeeded() async {
guard AppGroup.isAvailable, !isActive else { return }
let markedActive = AppGroup.defaults.bool(forKey: FlowSessionKeys.flowSessionActive)
guard markedActive, let remaining = FlowSessionBridge.remainingSessionDuration(), remaining > 0 else {
if markedActive {
FlowSessionBridge.markSessionInactive()
}
return
}
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
return
}
isStarting = true
sessionWarning = nil
defer { isStarting = false }
do {
try capture.start()
} catch {
let message = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
sessionWarning = message
FlowSessionBridge.markSessionInactive()
debug("bootstrap capture failed: \(message)")
return
}
FlowSessionBridge.writeHeartbeat()
FlowSessionDarwin.postSessionChanged()
isActive = true
ScreenWakeLock.acquire()
if let expires = FlowSessionBridge.sessionExpiresAt() {
sessionExpiresAt = Date(timeIntervalSince1970: expires)
}
startHeartbeat()
startPolling()
startLevelPublishing()
scheduleExpiry(after: remaining)
// v0.2.0: iOS `SpeechAnalyzer` needs no warm-up. We still
// re-bind the cached `sessionASR` so a config flip mid-session
// (e.g. switching from cloud to local) is honoured.
bindSessionASRIfNeeded()
scheduleASRWarmup()
FlowLiveActivityController.startSession()
debug("Flow session restored (\(Int(remaining))s remaining)")
}
func endSession() {
guard isActive else { return }
debug("Flow session ended")
+5 -2
View File
@@ -37,8 +37,11 @@ struct FlowColdStartOverlay: View {
.ignoresSafeArea()
VStack(spacing: Spacing.xl) {
Image(systemName: "waveform.circle.fill")
.font(.system(size: 56))
Image("OSGBrandMark")
.renderingMode(.template)
.resizable()
.aspectRatio(contentMode: .fit)
.frame(width: 64, height: 64)
.foregroundStyle(palette.accent)
.accessibilityHidden(true)
+19
View File
@@ -36,6 +36,13 @@ struct HomeView: View {
micStatus != .granted || speechStatus != .granted
}
/// Can the user start a session right now from the Home footer? Only when
/// nothing is live/starting and permissions are already granted (otherwise
/// the permission guidance card is the correct call to action).
private var canManuallyStartSession: Bool {
!sessionIsLive && !needsPermissionSetup
}
private var shouldShowKeyboardHint: Bool {
!keyboardHintDismissed
&& !KeyboardSetupBridge.isReadyForOnboardingSkip
@@ -198,6 +205,18 @@ struct HomeView: View {
}
.buttonStyle(.plain)
.padding(.leading, Spacing.xs)
} else if canManuallyStartSession {
// Sessions auto-start on foreground, but after a manual stop /
// expiry the user needs a reliable, no-jump way back in.
Button {
flowManager.activateOnForeground()
} label: {
Text("home.flow.startShort")
.font(TypeStyle.caption2)
.foregroundStyle(palette.accent)
}
.buttonStyle(.plain)
.padding(.leading, Spacing.xs)
}
}
.fixedSize(horizontal: true, vertical: false)
+3 -3
View File
@@ -37,7 +37,7 @@ struct MainAppRoot: View {
.animation(.easeInOut(duration: 0.2), value: flowManager.coldStartContext != nil)
.onAppear {
flowManager.setAppForeground(scenePhase == .active)
flowManager.restoreSessionIfNeeded()
flowManager.activateOnForeground()
}
.onReceive(NotificationCenter.default.publisher(for: .osgKeyboardOpenURL)) { notification in
guard let url = notification.userInfo?["url"] as? URL else { return }
@@ -45,13 +45,13 @@ struct MainAppRoot: View {
}
.onChange(of: config.hasCompletedOnboarding) { _, done in
if done {
flowManager.warmupAfterOnboardingIfNeeded()
flowManager.activateOnForeground()
}
}
.onChange(of: scenePhase) { _, phase in
flowManager.handleScenePhase(phase)
guard phase == .active, config.hasCompletedOnboarding else { return }
flowManager.restoreSessionIfNeeded()
flowManager.activateOnForeground()
}
}
+25
View File
@@ -296,6 +296,7 @@
"home.flow.start" = "Start voice session";
"home.flow.end" = "End voice session";
"home.flow.endShort" = "End";
"home.flow.startShort" = "Start";
"home.preview.label" = "Try typing";
"home.preview.placeholder" = "Tap to type and test…";
"home.stats.dictationDuration" = "Dictation time";
@@ -405,3 +406,27 @@
"hostApp.chrome" = "Chrome";
"hostApp.weibo" = "Weibo";
"hostApp.xiaohongshu" = "RED";
"hostApp.signal" = "Signal";
"hostApp.kakaotalk" = "KakaoTalk";
"hostApp.viber" = "Viber";
"hostApp.zalo" = "Zalo";
"hostApp.skype" = "Skype";
"hostApp.zoom" = "Zoom";
"hostApp.things" = "Things";
"hostApp.todoist" = "Todoist";
"hostApp.evernote" = "Evernote";
"hostApp.onenote" = "OneNote";
"hostApp.spark" = "Spark";
"hostApp.firefox" = "Firefox";
"hostApp.edge" = "Edge";
"hostApp.facebook" = "Facebook";
"hostApp.instagram" = "Instagram";
"hostApp.x" = "X";
"hostApp.threads" = "Threads";
"hostApp.snapchat" = "Snapchat";
"hostApp.reddit" = "Reddit";
"hostApp.pinterest" = "Pinterest";
"hostApp.zhihu" = "Zhihu";
"hostApp.bilibili" = "Bilibili";
"hostApp.douyin" = "Douyin";
"hostApp.tiktok" = "TikTok";
@@ -295,6 +295,7 @@
"home.flow.start" = "启动语音会话";
"home.flow.end" = "结束语音会话";
"home.flow.endShort" = "结束";
"home.flow.startShort" = "开启";
"home.preview.label" = "输入测试";
"home.preview.placeholder" = "点这里试试键盘";
"home.stats.dictationDuration" = "听写时长";
@@ -404,3 +405,27 @@
"hostApp.chrome" = "Chrome";
"hostApp.weibo" = "微博";
"hostApp.xiaohongshu" = "小红书";
"hostApp.signal" = "Signal";
"hostApp.kakaotalk" = "KakaoTalk";
"hostApp.viber" = "Viber";
"hostApp.zalo" = "Zalo";
"hostApp.skype" = "Skype";
"hostApp.zoom" = "Zoom";
"hostApp.things" = "Things";
"hostApp.todoist" = "Todoist";
"hostApp.evernote" = "印象笔记";
"hostApp.onenote" = "OneNote";
"hostApp.spark" = "Spark";
"hostApp.firefox" = "Firefox";
"hostApp.edge" = "Edge";
"hostApp.facebook" = "Facebook";
"hostApp.instagram" = "Instagram";
"hostApp.x" = "X";
"hostApp.threads" = "Threads";
"hostApp.snapchat" = "Snapchat";
"hostApp.reddit" = "Reddit";
"hostApp.pinterest" = "Pinterest";
"hostApp.zhihu" = "知乎";
"hostApp.bilibili" = "哔哩哔哩";
"hostApp.douyin" = "抖音";
"hostApp.tiktok" = "TikTok";
+18 -26
View File
@@ -1,9 +1,18 @@
// HostAppLauncher.swift
// OSGKeyboard · Keyboard Extension
//
// Opens the host app via URL using extension-safe strategies:
// 1. `extensionContext.open` (official)
// 2. Responder-chain `UIApplication.open` (TypeWhisper pattern)
// Opens the host app from the keyboard extension.
//
// Reality check (verified against iOS 1826 behaviour):
// `extensionContext.open` is documented for Today widgets only; for a
// keyboard extension it resolves `false`, so we do not use it.
// The deprecated `openURL:` selector hack was disabled in iOS 18
// ("BUG IN CLIENT OF UIKIT migrate to open(_:options:completionHandler:)").
// The still-working path is: walk the responder chain to `UIApplication`
// and call the non-deprecated `open(_:options:completionHandler:)`. This
// requires Full Access and grows less reliable on newer iOS, so we report
// the *real* success from the completion handler instead of assuming it
// worked callers degrade to on-keyboard guidance when it returns false.
import UIKit
@@ -14,34 +23,17 @@ enum HostAppLauncher {
from controller: KeyboardViewController,
completion: @escaping @MainActor (Bool) -> Void
) {
if let context = controller.extensionContext {
context.open(url) { success in
Task { @MainActor in
if success {
completion(true)
return
}
completion(openViaResponderChain(url, from: controller))
}
}
return
}
completion(openViaResponderChain(url, from: controller))
}
@MainActor
private static func openViaResponderChain(
_ url: URL,
from controller: KeyboardViewController
) -> Bool {
var responder: UIResponder? = controller
while let current = responder {
if let application = current as? UIApplication {
application.open(url, options: [:]) { _ in }
return true
application.open(url, options: [:]) { success in
Task { @MainActor in completion(success) }
}
return
}
responder = current.next
}
return false
// No `UIApplication` in the responder chain cannot open the host app.
completion(false)
}
}
@@ -61,12 +61,18 @@ final class KeyboardFlowCoordinator {
isPendingFlowStart || isFlowRecording || isAwaitingFlowResult
}
/// Session/transcription changes are pushed in real time by Darwin
/// notifications (see `KeyboardConfigSync.installDarwinObservers`), so this
/// loop is only a low-frequency safety net for coalesced/dropped Darwin
/// signals hence 3 s rather than 1 Hz to save battery while idle.
private static let sessionMonitorIntervalNs: UInt64 = 3_000_000_000
func startSessionMonitor() {
flowSessionMonitorTask?.cancel()
flowSessionMonitorTask = Task { @MainActor [weak self] in
while !Task.isCancelled {
self?.refreshSessionState()
try? await Task.sleep(nanoseconds: 1_000_000_000)
try? await Task.sleep(nanoseconds: Self.sessionMonitorIntervalNs)
}
}
}
@@ -168,8 +174,14 @@ final class KeyboardFlowCoordinator {
debug("openHostApp path=\(path) success=\(success)")
guard !success else { return }
// The open genuinely failed (iOS blocked it / no Full Access). Don't
// let the 30s watchdog spin cancel the pending start immediately and
// guide the user to open OSGKeyboard manually.
if path == "startflow", isPendingFlowStart {
state.lastTranscript = ExtL10n.string("keyboard.flow.manualOpenHost")
isPendingFlowStart = false
flowStartDeadline = 0
stopFlowWatchdog()
showManualOpenHint(path: "startflow")
return
}
+4 -15
View File
@@ -142,8 +142,7 @@ public struct KeyboardRootView: View {
micDisabled: state.micDisabled,
micDisabledHint: state.micDisabledHint,
cursorDragHintActive: state.cursorDragActive,
openSettings: state.openSettings,
startFlowSession: state.startFlowSession
openSettings: state.openSettings
)
.frame(height: KeyboardLayoutMetrics.transcriptLineHeight)
}
@@ -336,7 +335,6 @@ private struct TranscriptLine: View {
let micDisabledHint: String
let cursorDragHintActive: Bool
let openSettings: () -> Void
let startFlowSession: () -> Void
var body: some View {
ZStack {
@@ -366,18 +364,9 @@ private struct TranscriptLine: View {
.font(TypeStyle.caption)
.foregroundStyle(palette.textTertiary)
} else {
HStack(spacing: 6) {
ExtL10n.text("keyboard.flow.sessionInactive")
.font(TypeStyle.caption)
.foregroundStyle(palette.textTertiary)
Button(action: startFlowSession) {
ExtL10n.text("keyboard.flow.start")
.font(TypeStyle.caption)
.foregroundStyle(palette.accent)
}
.buttonStyle(.plain)
.accessibilityHint(ExtL10n.text("keyboard.flow.startA11y"))
}
ExtL10n.text("keyboard.flow.sessionInactive")
.font(TypeStyle.caption)
.foregroundStyle(palette.textTertiary)
}
case .requestingPermissions:
HStack(spacing: 6) {
@@ -0,0 +1,6 @@
{
"info" : {
"author" : "xcode",
"version" : 1
}
}
@@ -0,0 +1,15 @@
{
"images" : [
{
"filename" : "OSGLogo.svg",
"idiom" : "universal"
}
],
"info" : {
"author" : "xcode",
"version" : 1
},
"properties" : {
"preserves-vector-representation" : true
}
}
@@ -0,0 +1,7 @@
<svg width="1024" height="1024" viewBox="0 0 1024 1024" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M627.511 512.419C627.511 448.783 576.127 397.197 512.741 397.197C449.355 397.197 397.971 448.783 397.971 512.419C397.971 576.054 449.355 627.641 512.741 627.641V637.837C443.746 637.837 387.814 581.686 387.814 512.419C387.814 443.152 443.746 387 512.741 387C581.736 387 637.668 443.152 637.668 512.419C637.668 581.686 581.736 637.837 512.741 637.837V627.641C576.127 627.641 627.511 576.054 627.511 512.419Z" fill="white"/>
<path d="M56.2632 512.419C56.2632 581.686 112.195 637.837 181.19 637.837C250.185 637.837 306.117 581.686 306.117 512.419C306.117 443.152 250.185 387 181.19 387C112.195 387 56.2632 443.152 56.2632 512.419Z" fill="white"/>
<path d="M399.618 562.094L399.618 552.257L512.74 552.257L512.74 562.094L399.618 562.094Z" fill="white"/>
<path d="M512.741 483.398V473.561H625.864V483.398H512.741Z" fill="white"/>
<path d="M843.308 387.981C910.987 387.981 966.093 441.8 968.171 508.975H843.308V518.812H968.096C965.014 585.066 910.324 637.836 843.308 637.836C774.313 637.836 718.381 581.904 718.381 512.909C718.381 443.914 774.312 387.981 843.308 387.981ZM968.234 518.812H968.096C968.187 516.855 968.234 514.888 968.234 512.909C968.234 511.593 968.211 510.281 968.171 508.975H968.234V518.812Z" fill="white"/>
</svg>

After

Width:  |  Height:  |  Size: 1.3 KiB

@@ -61,33 +61,24 @@ private struct FlowLiveActivityLockScreenView: View {
Spacer(minLength: 0)
FlowLiveActivityTrailingGlyph(phase: phase)
}
.padding(.horizontal, 4)
// iOS Live Activity lock-screen content needs margins so the leading
// logo and trailing glyph don't touch the card edges.
.padding(.horizontal, 16)
.padding(.vertical, 12)
}
}
/// Branded mark used in compactLeading so users see OSGKeyboard, not the system mic icon.
/// Transparent white OSG glyphs render directly on the black Dynamic Island.
private struct FlowLiveActivityBrandMark: View {
let size: CGFloat
var body: some View {
ZStack {
Circle()
.fill(
LinearGradient(
colors: [
Color(red: 0.35, green: 0.55, blue: 1.0),
Color(red: 0.22, green: 0.38, blue: 0.92)
],
startPoint: .topLeading,
endPoint: .bottomTrailing
)
)
Image(systemName: "keyboard")
.font(.system(size: size * 0.48, weight: .semibold))
.foregroundStyle(.white)
}
.frame(width: size, height: size)
.accessibilityLabel("OSGKeyboard")
Image("OSGLogo")
.resizable()
.aspectRatio(contentMode: .fit)
.frame(width: size, height: size)
.accessibilityLabel("OSGKeyboard")
}
}
@@ -105,8 +96,10 @@ private struct FlowLiveActivityTrailingGlyph: View {
.progressViewStyle(.circular)
.tint(.white)
case .idle:
Image(systemName: "mic.fill")
.foregroundStyle(.secondary)
// Session ready but NOT listening avoid a mic glyph so users
// don't think the keyboard is recording in the background.
Image(systemName: "checkmark.circle.fill")
.foregroundStyle(.green)
}
}
}
+1 -1
View File
@@ -3,7 +3,7 @@
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>en</string>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleDisplayName</key>
<string>OSGKeyboardLiveActivity</string>
<key>CFBundleExecutable</key>
@@ -163,6 +163,10 @@ public final class FlowContinuousCapture {
private var didInstallTap = false
private var isRunning = false
private var routeObserver: NSObjectProtocol?
private var interruptionObserver: NSObjectProtocol?
private let log = Logger(subsystem: "com.osgkeyboard.shared", category: "FlowCapture")
public init() {}
public var running: Bool { isRunning }
@@ -170,7 +174,16 @@ public final class FlowContinuousCapture {
/// Configure `.playAndRecord`, install a permanent input tap, start the engine.
public func start() throws {
guard !isRunning else { return }
try activateEngine()
isRunning = true
installSessionObservers()
}
/// Bring up the audio session + engine for the *current* hardware route.
/// Reused for route-change / interruption recovery, so it always rebuilds
/// the tap against the live hardware format (which changes when the user
/// plugs in AirPods or a wired headset mid-session).
private func activateEngine() throws {
let session = AVAudioSession.sharedInstance()
do {
try session.setCategory(
@@ -204,23 +217,22 @@ public final class FlowContinuousCapture {
throw StartError.converterCreateFailed
}
if !didInstallTap {
let utteranceFlag = isUtteranceActive
let relay = streamRelay
let preroll = prerollStore
let levels = levelStore
let tap = Self.makeAudioTapBlock(
converter: converter,
targetFormat: targetFormat,
hwFormat: hwFormat,
utteranceFlag: utteranceFlag,
levelStore: levels,
prerollStore: preroll,
streamRelay: relay
)
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
didInstallTap = true
// Rebuild the tap so its bound hardware format matches the new route.
if didInstallTap {
inputNode.removeTap(onBus: 0)
didInstallTap = false
}
let tap = Self.makeAudioTapBlock(
converter: converter,
targetFormat: targetFormat,
hwFormat: hwFormat,
utteranceFlag: isUtteranceActive,
levelStore: levelStore,
prerollStore: prerollStore,
streamRelay: streamRelay
)
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
didInstallTap = true
audioEngine.prepare()
do {
@@ -228,11 +240,11 @@ public final class FlowContinuousCapture {
} catch {
throw StartError.engineStartFailed(error.localizedDescription)
}
isRunning = true
}
/// Tear down the engine and release the audio session.
public func stop() {
removeSessionObservers()
isUtteranceActive.withLock { $0 = false }
streamRelay.finish()
@@ -266,6 +278,98 @@ public final class FlowContinuousCapture {
}
}
// MARK: - Route / interruption recovery
private func installSessionObservers() {
let center = NotificationCenter.default
if routeObserver == nil {
routeObserver = center.addObserver(
forName: AVAudioSession.routeChangeNotification,
object: nil,
queue: .main
) { [weak self] note in
// Extract Sendable primitives here (Notification isn't Sendable)
// before hopping onto the main actor.
let reasonRaw = note.userInfo?[AVAudioSessionRouteChangeReasonKey] as? UInt
MainActor.assumeIsolated { self?.handleRouteChange(reasonRaw: reasonRaw) }
}
}
if interruptionObserver == nil {
interruptionObserver = center.addObserver(
forName: AVAudioSession.interruptionNotification,
object: nil,
queue: .main
) { [weak self] note in
let typeRaw = note.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt
let optionsRaw = note.userInfo?[AVAudioSessionInterruptionOptionKey] as? UInt
MainActor.assumeIsolated {
self?.handleInterruption(typeRaw: typeRaw, optionsRaw: optionsRaw)
}
}
}
}
private func removeSessionObservers() {
let center = NotificationCenter.default
if let routeObserver { center.removeObserver(routeObserver) }
if let interruptionObserver { center.removeObserver(interruptionObserver) }
routeObserver = nil
interruptionObserver = nil
}
private func handleRouteChange(reasonRaw: UInt?) {
guard isRunning else { return }
guard let reasonRaw,
let reason = AVAudioSession.RouteChangeReason(rawValue: reasonRaw) else { return }
// Only rebuild for real device swaps (plugging / unplugging a headset
// or AirPods). Ignore `.categoryChange`, which we trigger ourselves.
switch reason {
case .oldDeviceUnavailable, .newDeviceAvailable:
log.info("Audio route changed (\(reasonRaw, privacy: .public)) — rebuilding engine")
rebuildEngine()
default:
break
}
}
private func handleInterruption(typeRaw: UInt?, optionsRaw: UInt?) {
guard let typeRaw,
let type = AVAudioSession.InterruptionType(rawValue: typeRaw) else { return }
switch type {
case .began:
// The system already paused our engine; wait for `.ended`.
log.info("Audio interruption began")
case .ended:
guard isRunning else { return }
let shouldResume: Bool
if let optionsRaw {
shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsRaw).contains(.shouldResume)
} else {
shouldResume = true
}
if shouldResume {
log.info("Audio interruption ended — resuming capture")
rebuildEngine()
}
@unknown default:
break
}
}
/// Stop and rebuild the engine against the current route, keeping
/// `isRunning` intact so the session survives the swap transparently.
private func rebuildEngine() {
guard isRunning else { return }
if audioEngine.isRunning {
audioEngine.stop()
}
do {
try activateEngine()
} catch {
log.error("Engine rebuild failed: \(error.localizedDescription, privacy: .public)")
}
}
/// Begin forwarding downsampled buffers to ASR for one utterance.
public func beginUtterance() -> AsyncStream<AudioBufferSnapshot> {
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
@@ -156,6 +156,153 @@ public enum HostAppURLRegistry {
displayNameKey: "hostApp.xiaohongshu",
returnURLString: "xhsdiscover://",
tier: 4
),
// Tier 2 (cont.) global IM / calls
HostAppEntry(
bundleId: "org.whispersystems.signal",
displayNameKey: "hostApp.signal",
returnURLString: "sgnl://",
tier: 2
),
HostAppEntry(
bundleId: "com.iwilab.KakaoTalk",
displayNameKey: "hostApp.kakaotalk",
returnURLString: "kakaotalk://",
tier: 2
),
HostAppEntry(
bundleId: "com.viber",
displayNameKey: "hostApp.viber",
returnURLString: "viber://",
tier: 2
),
HostAppEntry(
bundleId: "com.vng.zaloapp",
displayNameKey: "hostApp.zalo",
returnURLString: "zalo://",
tier: 2
),
HostAppEntry(
bundleId: "com.skype.skype",
displayNameKey: "hostApp.skype",
returnURLString: "skype://",
tier: 2
),
HostAppEntry(
bundleId: "us.zoom.videomeetings",
displayNameKey: "hostApp.zoom",
returnURLString: "zoomus://",
tier: 2
),
// Tier 3 (cont.) notes / mail / browser
HostAppEntry(
bundleId: "com.culturedcode.ThingsiPhone",
displayNameKey: "hostApp.things",
returnURLString: "things://",
tier: 3
),
HostAppEntry(
bundleId: "com.todoist.ios",
displayNameKey: "hostApp.todoist",
returnURLString: "todoist://",
tier: 3
),
HostAppEntry(
bundleId: "com.evernote.iPhone.Evernote",
displayNameKey: "hostApp.evernote",
returnURLString: "evernote://",
tier: 3
),
HostAppEntry(
bundleId: "com.microsoft.onenote",
displayNameKey: "hostApp.onenote",
returnURLString: "onenote://",
tier: 3
),
HostAppEntry(
bundleId: "com.readdle.smartemail",
displayNameKey: "hostApp.spark",
returnURLString: "readdle-spark://",
tier: 3
),
HostAppEntry(
bundleId: "org.mozilla.ios.Firefox",
displayNameKey: "hostApp.firefox",
returnURLString: "firefox://",
tier: 3
),
HostAppEntry(
bundleId: "com.microsoft.msedge",
displayNameKey: "hostApp.edge",
returnURLString: "microsoft-edge://",
tier: 3
),
// Tier 4 (cont.) global / China social
HostAppEntry(
bundleId: "com.facebook.Facebook",
displayNameKey: "hostApp.facebook",
returnURLString: "fb://",
tier: 4
),
HostAppEntry(
bundleId: "com.burbn.instagram",
displayNameKey: "hostApp.instagram",
returnURLString: "instagram://",
tier: 4
),
HostAppEntry(
bundleId: "com.atebits.Tweetie2",
displayNameKey: "hostApp.x",
returnURLString: "twitter://",
tier: 4
),
HostAppEntry(
bundleId: "com.burbn.barcelona",
displayNameKey: "hostApp.threads",
returnURLString: "barcelona://",
tier: 4
),
HostAppEntry(
bundleId: "com.toyopagroup.picaboo",
displayNameKey: "hostApp.snapchat",
returnURLString: "snapchat://",
tier: 4
),
HostAppEntry(
bundleId: "com.reddit.Reddit",
displayNameKey: "hostApp.reddit",
returnURLString: "reddit://",
tier: 4
),
HostAppEntry(
bundleId: "pinterest",
displayNameKey: "hostApp.pinterest",
returnURLString: "pinterest://",
tier: 4
),
HostAppEntry(
bundleId: "com.zhihu.ios",
displayNameKey: "hostApp.zhihu",
returnURLString: "zhihu://",
tier: 4
),
HostAppEntry(
bundleId: "tv.danmaku.bili",
displayNameKey: "hostApp.bilibili",
returnURLString: "bilibili://",
tier: 4
),
HostAppEntry(
bundleId: "com.ss.iphone.ugc.Aweme",
displayNameKey: "hostApp.douyin",
returnURLString: "snssdk1128://",
tier: 4
),
HostAppEntry(
bundleId: "com.zhiliaoapp.musically",
displayNameKey: "hostApp.tiktok",
returnURLString: "tiktok://",
tier: 4
)
]
+4 -3
View File
@@ -18,7 +18,7 @@
OSGKeyboard is a free, source-available alternative to commercial voice-input tools. It runs as a **Custom Keyboard Extension** on iOS, so you can use it in **any app** — Messages, Notes, Mail, WeChat, ChatGPT, Claude, Cursor, you name it.
1. Tap the mic to start recording
2. Speak naturally (up to 60 seconds per take)
2. Speak naturally (up to 3.5 minutes / 210 seconds per take)
3. Tap again to stop — the AI polishes your words into clean text and inserts at the cursor
Audio is transcribed **on-device** by Apple's `SpeechAnalyzer` + `DictationTranscriber` (iOS 26+). Only the **polished transcript** is sent to your chosen cloud LLM. **No audio ever leaves your phone.**
@@ -29,7 +29,7 @@ Under the hood, OSGKeyboard uses a **Flow session model**: a long-lived audio se
## Features
- 🎙 **Tap-to-toggle recording** with a Typeless-style circular mic button, 60-second per-take cap with live countdown
- 🎙 **Tap-to-toggle recording** with a Typeless-style circular mic button, 3.5-minute (210s) per-take cap with live countdown
- 🧠 **On-device ASR** (`SpeechAnalyzer` + `DictationTranscriber`, iOS 26+)
- ✍️ **AI polishing** — adds structure, punctuation, fixes grammar, optionally produces lists
- 🧩 **Local + cloud polish toggle** — local engine is ASR-only by default; opt into a post-ASR cloud polish step (DeepSeek by default) when the iOS speech recognition isn't strong enough for your environment (noisy far-field audio, strong accents, etc.)
@@ -168,7 +168,8 @@ To set it as the new default for first-time users, also bump the `defaultProvide
- **~60 MB memory cap** for the keyboard extension (iOS sandbox). The Flow session is hosted in the main app, so audio buffers and ASR models live there, not in the extension.
- **"Allow Full Access" required.** Without it, the keyboard can't reach the microphone or make network requests for cloud polish.
- **Password fields and some `WKWebView` textareas** are blocked by iOS itself — not something we can work around.
- **60-second per-take cap.** A long take is automatically stopped and dispatched for transcription; a new take can be started immediately.
- **3.5-minute (210s) per-take cap.** A long take is automatically stopped and dispatched for transcription; a new take can be started immediately.
- **Force-quitting the host app does not resurrect the old session.** The Live Activity is cleared immediately; the next time you open the app (with permissions granted) a fresh voice session starts automatically.
- **3-minute per-utterance ASR cap.** If you exceed it, the pipeline gracefully splits into multiple stitched chunks.
- **No on-device LLM polish.** The local engine is ASR-only; "AI polish" is always cloud-based and configurable. On-device model support was explored in v0.2.0 and rolled back in v0.2.1 to keep the dependency surface at zero SPM packages.
- **URL scheme `osgkeyboard://`** can be opened by any app on the device. We don't trust it for anything beyond "wake the host app and (re)start the Flow session"; it never carries your API key or other secrets.
+4 -3
View File
@@ -17,7 +17,7 @@
OSGKeyboard 是商业语音输入工具的免费、源码可见替代方案。它以 **iOS 自定义键盘扩展** 的形式运行,所以你可以在 **任何 App** 里使用 —— 微信、备忘录、邮件、ChatGPT、Claude、Cursor,无所不能。
1. 按下麦克风键开始录音
2. 自由说话(单次上限 60 秒)
2. 自由说话(单次上限 3.5 分钟 / 210 秒)
3. 再按一下结束 —— AI 自动整理成干净的文字并插入光标
**音频始终在设备本地转写**iOS 26+ 的 `SpeechAnalyzer` + `DictationTranscriber`),**只有润色后的文本** 会发到你选择的云端 LLM。**音频永不离开你的手机。**
@@ -28,7 +28,7 @@ OSGKeyboard 是商业语音输入工具的免费、源码可见替代方案。
## 特性
- 🎙 **点按录音** —— Typeless 风格的圆形麦克风按钮,单次上限 60 秒并实时倒计时
- 🎙 **点按录音** —— Typeless 风格的圆形麦克风按钮,单次上限 3.5 分钟(210 秒并实时倒计时
- 🧠 **端侧 ASR**iOS 26+ `SpeechAnalyzer` + `DictationTranscriber`
- ✍️ **AI 润色** —— 自动加结构、补标点、修正语法、可生成列表
- 🧩 **本地 + 云端润色开关** —— 本地模式默认仅在设备上识别;若 iOS 语音识别效果不理想(远场、噪声、方言),可开启「识别后云端润色」,默认走 DeepSeek
@@ -165,7 +165,8 @@ LLMProvider(
- **键盘扩展约 60 MB 内存上限**(iOS 沙盒)。Flow 会话由主 App 承载,音频缓冲与 ASR 模型都在主 App 侧,不占用扩展内存。
- **必须「允许完全访问」**。否则键盘无法使用麦克风,也无法发起云端润色请求。
- **密码框与部分 `WKWebView` 输入框不可用**(iOS 系统限制,无法绕过)。
- **单次录音上限 60 秒**。到点自动停止并提交识别,下次可立即开始新的录音。
- **单次录音上限 3.5 分钟(210 秒**。到点自动停止并提交识别,下次可立即开始新的录音。
- **杀掉主 App 后不复活旧会话**。灵动岛会立即清理;下次回到主 App 时(权限齐全)自动开启新的语音会话。
- **单次 utterance ASR 上限 3 分钟**。超出后会拆成多个 chunk 拼接识别。
- **不做端侧 LLM 润色**。本地引擎仅做 ASR"AI 润色"始终走云端、可配置。v0.2.0 曾尝试引入端侧模型,v0.2.1 回滚以保持零 SPM 依赖。
- **URL Scheme `osgkeyboard://`** 任何 App 都可调用。OSGKeyboard 只把它用于"唤醒主 App / 续期 Flow 会话"**不** 传递 API Key 等敏感信息。
+10 -10
View File
@@ -161,8 +161,8 @@
### Phase 4 新增
- [x] 打开 App 后 **自动** 语音会话(权限齐全时)
- [x] 杀 App 再开 → **冷启动恢复**(未过期
- [x] 键盘 **点按** 开始/结束;60s 倒计时 + 最后 10s 变红
- [x] 杀 App 再开 → 灵动岛立即清理,**回到 App 时自动开新会话**(不复活旧会话
- [x] 键盘 **点按** 开始/结束;**3.5 分钟(210s** 倒计时 + 最后 10s 变红
- [x] Onboarding **分步权限** 完整可走通
- [x] 隐私政策 URL 可访问;App 内可打开
- [ ] App Store 隐私标签与政策一致(A3 待人工)
@@ -176,8 +176,8 @@
2. 切备忘录 → **点按**键盘麦开始 → 再点结束 → **文字出现**
3. **不跳主 App**,重复 5 次
4. 本地模式 + 云端模式各测 1 次
5. 60s 倒计时 + 最后 10 秒变红;到点自动识别
6. 杀 App 再开 → 会话恢复(未过期时
5. 3.5 分钟(210s倒计时 + 最后 10 秒变红;到点自动识别
6. 杀 App 再开 → 灵动岛立即清理;回到 App 时自动开新会话(不复活旧会话)
7. 预览 sheet「开始/停止录音」仍可用(Flow 未启动时)
---
@@ -194,9 +194,9 @@
| 会话未启动 | 保持现有逻辑(拉 App / 提示去主 App) |
| 权限引导 | 欢迎 → 麦克风 → 语音识别 → 键盘+完全访问 → 引擎/API;**仅首次或权限未定时** |
| 自动开语音会话 | 进 App 且权限齐全 → **自动开**;有效会话 → **续期****不设关闭开关** |
| 冷启动恢复 | 参考 TypeWhisper `checkExistingSession` |
| 冷启动恢复 | **杀 App 不复活旧会话**;回到前台由 `activateOnForeground()` 清理孤儿灵动岛并自动开新会话(权限齐全时) |
| Home 按钮 | 自动开后 **隐藏「启动」**;**保留「结束」**;失败显示原因 + 去设置 |
| 单次录音上限 | **60s**;键盘到点 auto `stopped`;倒计时 **A(剩余)+ C(最后 10s 变红)**,显示在按钮内 |
| 单次录音上限 | **210s3.5 分钟)**;键盘到点 auto `stopped`;倒计时 **A(剩余)+ C(最后 10s 变红)**,显示在按钮内 |
| 隐私政策 URL | **GitHub Pages**(仓库站点,如 `…/privacy` |
| 自动开会话开关 | **先不加** |
@@ -226,13 +226,13 @@
- [x] **C3** Home 卡片 UX:进行中 + 结束;失败原因 + 去设置;隐藏手动「启动」
- [x] **C4**`scenePhase.active` 续期对齐
### 7.5 批次 D · 键盘点按录音 + 60s 倒计时
### 7.5 批次 D · 键盘点按录音 + 3.5 分钟(210s倒计时
- [x] **D1** `RecordButton` 改为 toggle(替换长按手势)
- [x] **D2** 识别中禁用按钮
- [x] **D3** 按钮内剩余时间倒计时(`M:SS`
- [x] **D4** 最后 10 秒变红/橙(A+C
- [x] **D5** 60s 到点自动 `stopped` → 「识别中…」
- [x] **D5** 210s3.5 分钟)到点自动 `stopped` → 「识别中…」
- [x] **D6** 无障碍 / 占位文案改为「点按说话」类
### 7.6 批次 E · 多语言完善
@@ -244,12 +244,12 @@
### 7.7 批次 F · Phase 3 收尾
- [x] **F1** B3:会话过期键盘提示、Darwin(可选)、路由(可选)
- [ ] **F2** B4:全场景回归(含自动开、60s、杀 App 恢复)— 待真机
- [ ] **F2** B4:全场景回归(含自动开、210s、杀 App 不复活/回前台自动开)— 待真机
- [x] **F3** 更新 §5 验收勾选
### 7.8 任务追踪(Phase 4
- [x] P4-0:产品规格拍板(交互、权限、自动会话、60s、隐私 URL
- [x] P4-0:产品规格拍板(交互、权限、自动会话、210s、隐私 URL
- [ ] P4-A:上架合规批次(A1/A2/A5 代码侧已完成;A3/A4 待人工)
- [x] P4-B:权限引导
- [x] P4-C:会话自动化
+24
View File
@@ -140,6 +140,30 @@ targets:
- googlechrome
- sinaweibo
- xhsdiscover
- sgnl
- kakaotalk
- viber
- zalo
- skype
- zoomus
- things
- todoist
- evernote
- onenote
- readdle-spark
- firefox
- microsoft-edge
- fb
- instagram
- twitter
- barcelona
- snapchat
- reddit
- pinterest
- zhihu
- bilibili
- snssdk1128
- tiktok
# All network calls are HTTPS (NSAppTransportSecurity above);
# nothing in the app uses non-exempt encryption. Declaring
# `ITSAppUsesNonExemptEncryption: false` lets us skip the