rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(flow): add Picture in Picture keep-alive mode

Browse Source 
- Add FlowKeepAliveMode (Dynamic Island vs PiP) with iCloud sync
- Settings: keep-alive picker; inactivity timeout only for Live Activity
- PiP: waveform sample-buffer controller, mic released between utterances
- PiP sessions have no idle expiry; user closing PiP ends the session
- FlowSessionManager branches hostReady, session start, and utterance paths
- UIBackgroundModes picture-in-picture; bilingual strings and tests

Co-authored-by: Rocky <hkgood@users.noreply.github.com>
This commit is contained in:
Cursor Agent
2026-07-26 10:34:52 +00:00
parent e6f99d2744
commit 70ba3a4359
18 changed files with 801 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OSGKeyboard/Services/FlowPictureInPictureController.swift
+307
View File
@@ -0,0 +1,307 @@
// FlowPictureInPictureController.swift
// OSGKeyboard · Main App
//
// PiP keep-alive for Flow sessions: enqueues live waveform sample buffers
// so the host process stays eligible for multitasking while the mic is off
// between utterances.
import AVFoundation
import AVKit
import CoreMedia
import UIKit
@MainActor
final class FlowPictureInPictureController: NSObject {
/// User closed the PiP window — host should end the Flow session.
var onUserDismissed: (() -> Void)?
private(set) var isPictureInPictureActive = false
let displayLayer = AVSampleBufferDisplayLayer()
private var pipController: AVPictureInPictureController?
private var displayLink: CADisplayLink?
private weak var hostView: UIView?
private var waveformLevels: [Float] = Array(repeating: 0, count: 24)
private var isStoppingProgrammatically = false
private var frameIndex: Int64 = 0
// MARK: - Host view
func attachHostView(_ view: UIView) {
hostView = view
displayLayer.frame = view.bounds
displayLayer.videoGravity = .resizeAspectFill
displayLayer.removeFromSuperlayer()
view.layer.addSublayer(displayLayer)
configureControllerIfNeeded()
}
func updateHostLayoutIfNeeded() {
guard let hostView else { return }
displayLayer.frame = hostView.bounds
}
// MARK: - Lifecycle
@discardableResult
func start() -> Bool {
guard AVPictureInPictureController.isPictureInPictureSupported() else {
return false
}
configureControllerIfNeeded()
startFramePump()
guard pipController != nil else { return false }
if pipController?.isPictureInPictureActive == true {
isPictureInPictureActive = true
return true
}
enqueueWaveformFrame()
pipController?.startPictureInPicture()
return true
}
func startAndWait(timeout: TimeInterval = 4) async -> Bool {
if isPictureInPictureActive { return true }
guard start() else { return false }
let deadline = Date().addingTimeInterval(timeout)
while Date() < deadline {
if isPictureInPictureActive { return true }
if pipController?.isPictureInPictureActive == true {
isPictureInPictureActive = true
return true
}
try? await Task.sleep(nanoseconds: 50_000_000)
}
return isPictureInPictureActive
}
func stop() {
isStoppingProgrammatically = true
stopFramePump()
pipController?.stopPictureInPicture()
displayLayer.flushAndRemoveImage()
isPictureInPictureActive = false
isStoppingProgrammatically = false
}
func updateWaveformLevels(_ levels: [Float]) {
guard !levels.isEmpty else { return }
waveformLevels = levels
}
// MARK: - Private
private func configureControllerIfNeeded() {
guard pipController == nil else { return }
guard AVPictureInPictureController.isPictureInPictureSupported() else { return }
let contentSource = AVPictureInPictureController.ContentSource(
sampleBufferDisplayLayer: displayLayer,
playbackDelegate: self
)
let controller = AVPictureInPictureController(contentSource: contentSource)
controller.delegate = self
controller.canStartPictureInPictureAutomaticallyFromInline = true
pipController = controller
}
private func startFramePump() {
guard displayLink == nil else { return }
let link = CADisplayLink(target: self, selector: #selector(handleDisplayLink(_:)))
link.preferredFrameRateRange = CAFrameRateRange(minimum: 20, maximum: 30, preferred: 24)
link.add(to: .main, forMode: .common)
displayLink = link
}
private func stopFramePump() {
displayLink?.invalidate()
displayLink = nil
}
@objc private func handleDisplayLink(_ link: CADisplayLink) {
enqueueWaveformFrame()
updateHostLayoutIfNeeded()
if let pipController, !pipController.isPictureInPictureActive, pipController.isPictureInPicturePossible {
pipController.startPictureInPicture()
}
}
private func enqueueWaveformFrame() {
guard let sampleBuffer = makeWaveformSampleBuffer(levels: resolvedLevels()) else { return }
if displayLayer.status == .failed {
displayLayer.flush()
}
displayLayer.enqueue(sampleBuffer)
}
private func resolvedLevels() -> [Float] {
if waveformLevels.contains(where: { $0 > 0.02 }) {
return waveformLevels
}
// Idle breathing animation between utterances.
let phase = Float(frameIndex) * 0.12
return (0..<waveformLevels.count).map { index in
let wave = sin(phase + Float(index) * 0.45)
return max(0.04, 0.04 + wave * 0.03)
}
}
private func makeWaveformSampleBuffer(levels: [Float]) -> CMSampleBuffer? {
let width = 320
let height = 180
frameIndex += 1
var pixelBuffer: CVPixelBuffer?
let attrs: [String: Any] = [
kCVPixelBufferCGImageCompatibilityKey as String: true,
kCVPixelBufferCGBitmapContextCompatibilityKey as String: true,
]
let status = CVPixelBufferCreate(
kCFAllocatorDefault,
width,
height,
kCVPixelFormatType_32BGRA,
attrs as CFDictionary,
&pixelBuffer
)
guard status == kCVReturnSuccess, let pixelBuffer else { return nil }
CVPixelBufferLockBaseAddress(pixelBuffer, [])
defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) }
guard let base = CVPixelBufferGetBaseAddress(pixelBuffer) else { return nil }
let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
let colorSpace = CGColorSpaceCreateDeviceRGB()
guard let context = CGContext(
data: base,
width: width,
height: height,
bitsPerComponent: 8,
bytesPerRow: bytesPerRow,
space: colorSpace,
bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue
) else { return nil }
// Dark backdrop + accent waveform bars.
context.setFillColor(UIColor(red: 0.07, green: 0.09, blue: 0.11, alpha: 1).cgColor)
context.fill(CGRect(x: 0, y: 0, width: width, height: height))
let barCount = max(levels.count, 1)
let barWidth = CGFloat(width) / CGFloat(barCount * 2)
let accent = UIColor(red: 0.20, green: 0.78, blue: 0.55, alpha: 1)
context.setFillColor(accent.cgColor)
for (index, level) in levels.enumerated() {
let clamped = CGFloat(min(max(level, 0), 1))
let barHeight = max(6, clamped * CGFloat(height) * 0.72)
let x = (CGFloat(index) * 2 + 0.5) * barWidth
let rect = CGRect(
x: x,
y: (CGFloat(height) - barHeight) / 2,
width: barWidth,
height: barHeight
)
let path = UIBezierPath(roundedRect: rect, cornerRadius: barWidth * 0.35)
context.addPath(path.cgPath)
context.fillPath()
}
var formatDescription: CMFormatDescription?
CMVideoFormatDescriptionCreateForImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: pixelBuffer,
formatDescriptionOut: &formatDescription
)
guard let formatDescription else { return nil }
var timing = CMSampleTimingInfo(
duration: CMTime(value: 1, timescale: 24),
presentationTimeStamp: CMTime(value: frameIndex, timescale: 24),
decodeTimeStamp: .invalid
)
var sampleBuffer: CMSampleBuffer?
CMSampleBufferCreateForImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: pixelBuffer,
dataReady: true,
makeDataReadyCallback: nil,
refcon: nil,
formatDescription: formatDescription,
sampleTiming: &timing,
sampleBufferOut: &sampleBuffer
)
return sampleBuffer
}
}
// MARK: - AVPictureInPictureControllerDelegate
extension FlowPictureInPictureController: AVPictureInPictureControllerDelegate {
func pictureInPictureControllerDidStartPictureInPicture(
_ pictureInPictureController: AVPictureInPictureController
) {
isPictureInPictureActive = true
}
func pictureInPictureControllerDidStopPictureInPicture(
_ pictureInPictureController: AVPictureInPictureController
) {
isPictureInPictureActive = false
stopFramePump()
guard !isStoppingProgrammatically else { return }
onUserDismissed?()
}
func pictureInPictureController(
_ pictureInPictureController: AVPictureInPictureController,
restoreUserInterfaceForPictureInPictureStopWithCompletionHandler completionHandler: @escaping (Bool) -> Void
) {
completionHandler(true)
}
}
// MARK: - AVPictureInPictureSampleBufferPlaybackDelegate
extension FlowPictureInPictureController: AVPictureInPictureSampleBufferPlaybackDelegate {
func pictureInPictureController(
_ pictureInPictureController: AVPictureInPictureController,
setPlaying playing: Bool
) {
if playing {
startFramePump()
} else {
stopFramePump()
}
}
func pictureInPictureControllerTimeRangeForPlayback(
_ pictureInPictureController: AVPictureInPictureController
) -> CMTimeRange {
CMTimeRange(start: .zero, duration: CMTime(value: 3600, timescale: 1))
}
func pictureInPictureControllerIsPlaybackPaused(
_ pictureInPictureController: AVPictureInPictureController
) -> Bool {
false
}
func pictureInPictureController(
_ pictureInPictureController: AVPictureInPictureController,
didTransitionToRenderSize newRenderSize: CMVideoDimensions
) {}
func pictureInPictureController(
_ pictureInPictureController: AVPictureInPictureController,
skipByInterval skipInterval: CMTime,
completion completionHandler: @escaping () -> Void
) {
completionHandler()
}
}
OSGKeyboard/Services/FlowSessionManager.swift
+179 -13
View File
@@ -23,6 +23,7 @@ final class FlowSessionManager: ObservableObject {
@Published var coldStartContext: FlowColdStartContext?
private let capture = FlowContinuousCapture()
private let pipController = FlowPictureInPictureController()
private let store = AppGroupStore()
/// Cloud-engine polish; local engine runs through built-in DeepSeek polish.
private let polisher = PolishingService()
@@ -78,6 +79,14 @@ final class FlowSessionManager: ObservableObject {
private var coldStartRecoveryTask: Task<Void, Never>?
/// Initial proof window — cold mic sessions often need >2.5s after app switch.
private static let coldStartAudioProofTimeout: TimeInterval = 6
private var usesPiPKeepAlive: Bool {
FlowSessionPolicy.keepAliveMode() == .pictureInPicture
}
func attachPiPHostView(_ view: UIView) {
pipController.attachHostView(view)
}
/// Guards the once-per-process launch reconciliation (scene reconnects
/// recreate the `@StateObject`-owned manager within the same process).
private static var didRunLaunchReconciliation = false
@@ -122,6 +131,11 @@ final class FlowSessionManager: ObservableObject {
kind: .recognitionInterrupted
)
}
pipController.onUserDismissed = { [weak self] in
guard let self, self.isActive else { return }
self.debug("PiP dismissed by user — ending Flow session")
self.endSession()
}
FlowTerminationCoordinator.register(self)
}
@@ -276,7 +290,9 @@ final class FlowSessionManager: ObservableObject {
// hit the same audio-proof timeout.
coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = nil
if capture.running {
if usesPiPKeepAlive {
pipController.stop()
} else if capture.running {
capture.stop()
}
sessionASR?.cancel()
@@ -330,6 +346,7 @@ final class FlowSessionManager: ObservableObject {
if capture.running {
capture.stop()
}
pipController.stop()
endBackgroundKeepAlive()
ScreenWakeLock.release()
@@ -400,6 +417,7 @@ final class FlowSessionManager: ObservableObject {
isUtteranceProcessing = false
capture.stop()
pipController.stop()
endBackgroundKeepAlive()
ScreenWakeLock.release()
sessionASR = nil
@@ -416,6 +434,10 @@ final class FlowSessionManager: ObservableObject {
}
func extendSession(duration: TimeInterval? = nil) {
guard !usesPiPKeepAlive else {
refreshHostReady()
return
}
let resolved = duration ?? FlowSessionPolicy.sessionDuration()
FlowSessionBridge.extendSession(by: resolved)
sessionExpiresAt = Date().addingTimeInterval(resolved)
@@ -489,6 +511,10 @@ final class FlowSessionManager: ObservableObject {
private func reactivateCaptureIfNeeded() async {
guard isActive else { return }
if usesPiPKeepAlive, !isUtteranceRecording, !isUtteranceProcessing, !capture.running {
refreshHostReady()
return
}
// A system interruption (call / Siri) may be in progress. Probe it:
// `setActive(true)` inside `reassertIfRunning` fails while the
// interruption is live and succeeds once it ends — which also covers
@@ -559,12 +585,22 @@ final class FlowSessionManager: ObservableObject {
let pollingAlive = pollingTask != nil && pollingTask?.isCancelled != true
let hasRecentAudio = capture.engineHasRecentAudio(maxAge: 2)
let canAcceptUtterance = capture.engineIsLive
&& pollingAlive
&& hasRecentAudio
&& !isUtteranceRecording
&& !isUtteranceProcessing
&& sessionWarning == nil
let canAcceptUtterance: Bool
if usesPiPKeepAlive {
canAcceptUtterance = pipController.isPictureInPictureActive
&& pollingAlive
&& !isUtteranceRecording
&& !isUtteranceProcessing
&& sessionWarning == nil
&& !capture.isInterrupted
} else {
canAcceptUtterance = capture.engineIsLive
&& pollingAlive
&& hasRecentAudio
&& !isUtteranceRecording
&& !isUtteranceProcessing
&& sessionWarning == nil
}
let reason: FlowReadySnapshot.Reason
if canAcceptUtterance {
@@ -575,9 +611,11 @@ final class FlowSessionManager: ObservableObject {
reason = .recording
} else if isUtteranceProcessing {
reason = .processing
} else if !capture.engineIsLive {
} else if usesPiPKeepAlive, !pipController.isPictureInPictureActive {
reason = .starting
} else if !usesPiPKeepAlive, !capture.engineIsLive {
reason = .audioEngineNotLive
} else if !hasRecentAudio {
} else if !usesPiPKeepAlive, !hasRecentAudio {
reason = .waitingForAudioProof
} else {
reason = .starting
@@ -650,6 +688,11 @@ final class FlowSessionManager: ObservableObject {
/// custom keyboard extension sees green immediately.
func refreshForInlineKeyboardFocus() async {
guard isActive else { return }
if usesPiPKeepAlive {
refreshHostReady()
FlowSessionBridge.writeHeartbeat()
return
}
await reactivateCaptureIfNeeded()
refreshHostReady()
if !FlowSessionBridge.isHostReady() {
@@ -662,7 +705,7 @@ final class FlowSessionManager: ObservableObject {
/// Extend expiry after utterance completion based on the inactivity policy.
private func touchSessionActivity() {
guard isActive else { return }
guard isActive, !usesPiPKeepAlive else { return }
FlowSessionBridge.touchLastActivity()
if let expires = FlowSessionBridge.sessionExpiresAt() {
sessionExpiresAt = Date(timeIntervalSince1970: expires)
@@ -693,6 +736,25 @@ final class FlowSessionManager: ObservableObject {
return
}
if usesPiPKeepAlive {
let pipReady = await pipController.startAndWait()
guard pipReady else {
let message = AppL10n.string("flow.pip.error.unavailable")
sessionWarning = message
traceState("startSessionAsync.failed", extra: "reason=pipUnavailable")
FlowSessionBridge.setHostReady(false)
if isColdStartHandoff {
showColdStartAudioFailure(message: message)
}
debug("PiP keep-alive failed to start")
return
}
activateFlowSessionAfterPiPProof(duration: duration)
traceState("startSessionAsync.ready")
debug("Flow session started (PiP keep-alive), mic released between utterances")
return
}
do {
try capture.start()
} catch {
@@ -729,6 +791,31 @@ final class FlowSessionManager: ObservableObject {
debug("Flow session started (\(Int(duration ?? FlowSessionPolicy.sessionDuration()))s inactivity window), continuous capture running")
}
private func activateFlowSessionAfterPiPProof(duration: TimeInterval?) {
let sessionId = activeSessionId ?? UUID()
activeSessionId = sessionId
lastHandledCommandSeq = 0
FlowSessionBridge.markSessionActive(duration: duration, sessionId: sessionId)
FlowSessionDarwin.postSessionChanged()
isActive = true
ScreenWakeLock.acquire()
sessionExpiresAt = nil
startHeartbeat()
startCommandObserver()
startPolling()
startLevelPublishing()
expiryTask?.cancel()
expiryTask = nil
bindSessionASRIfNeeded()
scheduleASRWarmup()
FlowLiveActivityController.startSession()
refreshHostReady()
traceState("activateFlowSessionAfterPiPProof.done")
}
private func activateFlowSessionAfterAudioProof(duration: TimeInterval?) {
let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration()
let sessionId = activeSessionId ?? UUID()
@@ -756,6 +843,12 @@ final class FlowSessionManager: ObservableObject {
private func prepareExistingSessionForColdStartReturn() async {
guard isColdStartHandoff, isActive else { return }
if usesPiPKeepAlive {
sessionWarning = nil
refreshHostReady()
handleColdStartAfterSessionReady()
return
}
await reactivateCaptureIfNeeded()
guard await waitForAudioProof() else {
let message = AppL10n.string("flow.coldStart.error.audioTimeout")
@@ -809,7 +902,7 @@ final class FlowSessionManager: ObservableObject {
}
private func scheduleAutoReturnToHostIfNeeded(hostEntry: HostAppEntry?) {
let skipSwitch = FlowSessionPolicy.skipAppSwitch()
let skipSwitch = usesPiPKeepAlive || FlowSessionPolicy.skipAppSwitch()
guard skipSwitch, hostEntry != nil else { return }
Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: 450_000_000)
@@ -829,6 +922,21 @@ final class FlowSessionManager: ObservableObject {
coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = Task { @MainActor [weak self] in
guard let self else { return }
if self.usesPiPKeepAlive {
let recovered = await self.pipController.startAndWait()
self.traceState("coldStartRecovery.pip", extra: "recovered=\(recovered)")
guard !Task.isCancelled, self.isColdStartHandoff else { return }
if recovered {
if self.isActive {
self.refreshHostReady()
self.handleColdStartAfterSessionReady()
} else {
self.activateFlowSessionAfterPiPProof(duration: duration)
self.handleColdStartAfterSessionReady()
}
}
return
}
var recovered = false
for attempt in 1...3 {
guard !Task.isCancelled, self.isColdStartHandoff else { return }
@@ -1000,7 +1108,12 @@ final class FlowSessionManager: ObservableObject {
switch command.action {
case .startRecording:
guard !isUtteranceRecording, !isUtteranceProcessing else { return }
beginUtterance(utteranceId: command.utteranceId, commandSeq: command.commandSeq)
Task { @MainActor [weak self] in
await self?.handleStartRecordingCommand(
utteranceId: command.utteranceId,
commandSeq: command.commandSeq
)
}
case .stopRecording:
guard currentUtteranceId == command.utteranceId else { return }
if isUtteranceRecording {
@@ -1070,6 +1183,42 @@ final class FlowSessionManager: ObservableObject {
)
}
private func handleStartRecordingCommand(utteranceId: UUID?, commandSeq: Int64) async {
if usesPiPKeepAlive {
refreshHostReady()
let micReady = await ensureCaptureReadyForPiPUtterance()
guard micReady else {
failUtterance(
message: AppL10n.string("flow.coldStart.error.audioTimeout"),
kind: .audioUnavailable
)
return
}
}
beginUtterance(utteranceId: utteranceId, commandSeq: commandSeq)
}
private func ensureCaptureReadyForPiPUtterance() async -> Bool {
if capture.engineHasRecentAudio(maxAge: 2) {
return true
}
do {
try capture.start()
} catch {
debug("PiP utterance capture start failed: \(error.localizedDescription)")
return false
}
return await capture.awaitAudioFlowing(timeout: Self.coldStartAudioProofTimeout)
}
private func releaseCaptureAfterPiPUtteranceIfNeeded() {
guard usesPiPKeepAlive, capture.running else { return }
guard !isUtteranceRecording, !isUtteranceProcessing else { return }
capture.stop()
pipController.updateWaveformLevels([])
refreshHostReady()
}
private func beginUtterance(utteranceId: UUID? = nil, commandSeq: Int64 = 0) {
guard capture.engineHasRecentAudio(maxAge: 2) else {
traceState("beginUtterance.blocked", extra: "reason=audioNotRecent")
@@ -1207,6 +1356,10 @@ final class FlowSessionManager: ObservableObject {
guard let self else { return }
let drainReport = await self.capture.endUtteranceAndDrain()
FlowDiagnostics.logDrain(drainReport)
if self.usesPiPKeepAlive {
self.capture.stop()
self.pipController.updateWaveformLevels([])
}
await self.finalizeUtterance(
sessionId: drainingSessionId,
utteranceId: drainingUtteranceId,
@@ -1229,6 +1382,7 @@ final class FlowSessionManager: ObservableObject {
chunkedPipeline = nil
asr.cancel()
capture.cancelUtterance()
releaseCaptureAfterPiPUtteranceIfNeeded()
currentPartial = ""
lastFinal = ""
chunkWarnings = []
@@ -1255,6 +1409,7 @@ final class FlowSessionManager: ObservableObject {
chunkedPipeline = nil
asr.cancel()
capture.cancelUtterance()
releaseCaptureAfterPiPUtteranceIfNeeded()
currentPartial = ""
lastFinal = ""
chunkWarnings = []
@@ -1277,6 +1432,8 @@ final class FlowSessionManager: ObservableObject {
finalizeTask?.cancel()
finalizeTask = nil
chunkedPipeline = nil
capture.cancelUtterance()
releaseCaptureAfterPiPUtteranceIfNeeded()
currentPartial = ""
lastFinal = ""
chunkWarnings = []
@@ -1589,6 +1746,9 @@ final class FlowSessionManager: ObservableObject {
while !Task.isCancelled {
guard let self, self.isActive else { break }
let levels = self.capture.currentAudioLevels()
if self.usesPiPKeepAlive {
self.pipController.updateWaveformLevels(levels)
}
if levels.contains(where: { $0 > 0 }) {
FlowSessionBridge.storeAudioLevels(levels)
}
@@ -1612,7 +1772,12 @@ final class FlowSessionManager: ObservableObject {
while !Task.isCancelled {
guard let self else { break }
if self.isActive, !self.capture.engineIsLive {
await self.reactivateCaptureIfNeeded()
let shouldReassert = !self.usesPiPKeepAlive
|| self.isUtteranceRecording
|| self.isUtteranceProcessing
if shouldReassert {
await self.reactivateCaptureIfNeeded()
}
}
FlowSessionBridge.writeHeartbeat()
self.refreshHostReady()
@@ -1627,6 +1792,7 @@ final class FlowSessionManager: ObservableObject {
}
private func scheduleExpiry(after duration: TimeInterval) {
guard !usesPiPKeepAlive else { return }
expiryTask?.cancel()
expiryTask = Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: UInt64(duration * 1_000_000_000))
OSGKeyboard/Views/FlowPiPHostView.swift
+23
View File
@@ -0,0 +1,23 @@
// FlowPiPHostView.swift
// OSGKeyboard · Main App
//
// Hidden host for the PiP sample-buffer display layer (must live in the window hierarchy).
import SwiftUI
import UIKit
struct FlowPiPHostView: UIViewRepresentable {
let attach: (UIView) -> Void
func makeUIView(context: Context) -> UIView {
let view = UIView(frame: CGRect(x: 0, y: 0, width: 2, height: 2))
view.isUserInteractionEnabled = false
view.backgroundColor = .clear
attach(view)
return view
}
func updateUIView(_ uiView: UIView, context: Context) {
attach(uiView)
}
}
OSGKeyboard/Views/MainAppRoot.swift
+8
View File
@@ -37,6 +37,14 @@ struct MainAppRoot: View {
}
}
.animation(.easeInOut(duration: 0.2), value: flowManager.coldStartContext != nil)
.background {
FlowPiPHostView { view in
flowManager.attachPiPHostView(view)
}
.frame(width: 2, height: 2)
.opacity(0.001)
.allowsHitTesting(false)
}
.onAppear {
flowManager.setAppForeground(scenePhase == .active)
// Register the URL handler BEFORE the foreground auto-start.
OSGKeyboard/Views/SettingsView.swift
+87 -18
View File
@@ -29,6 +29,8 @@ struct SettingsView: View {
// Dynamic locale list loaded from SFSpeechRecognizer on first appear.
@State private var dynamicLocales: [(id: String, onDevice: Bool)] = []
@State private var showResetConfirmation = false
@State private var showActiveFlowSessionAlert = false
@State private var pendingKeepAliveMode: FlowKeepAliveMode?
// v0.2.0: no on-device model manager / pending download state —
// iOS `SpeechAnalyzer` ships with iOS 26 and needs nothing
// downloaded.
@@ -105,27 +107,41 @@ struct SettingsView: View {
VStack(alignment: .leading, spacing: SettingsListMetrics.sectionLabelSpacing) {
sectionHeader("settings.flow.title")
VStack(spacing: 0) {
Toggle(isOn: $config.flowSkipAppSwitch) {
VStack(alignment: .leading, spacing: Spacing.xxs) {
Text("settings.flow.skipAppSwitch.title")
.font(TypeStyle.body)
.foregroundStyle(palette.textPrimary)
Text("settings.flow.skipAppSwitch.subtitle")
.font(TypeStyle.caption2)
.foregroundStyle(palette.textTertiary)
}
}
.tint(palette.accent)
.settingsListRow()
Divider().background(palette.divider)
FlowInactivityPickerRow(
FlowKeepAliveModePickerRow(
selection: Binding(
get: { config.flowInactivityDuration },
set: { config.flowInactivityDuration = $0 }
get: { config.flowKeepAliveMode },
set: { newMode in
applyKeepAliveModeChange(newMode)
}
)
)
if config.flowKeepAliveMode == .liveActivity {
Divider().background(palette.divider)
FlowInactivityPickerRow(
selection: Binding(
get: { config.flowInactivityDuration },
set: { config.flowInactivityDuration = $0 }
)
)
Divider().background(palette.divider)
Toggle(isOn: $config.flowSkipAppSwitch) {
flowSkipAppSwitchLabel
}
.tint(palette.accent)
.settingsListRow()
} else {
Divider().background(palette.divider)
Text("settings.flow.keepAlive.pictureInPicture.note")
.font(TypeStyle.caption2)
.foregroundStyle(palette.textTertiary)
.frame(maxWidth: .infinity, alignment: .leading)
.settingsListRow()
}
}
.background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.xl, style: .continuous))
.overlay(
@@ -133,6 +149,34 @@ struct SettingsView: View {
.stroke(palette.divider, lineWidth: 0.5)
)
}
.alert("settings.flow.keepAlive.activeSession.title", isPresented: $showActiveFlowSessionAlert) {
Button("common.done", role: .cancel) {
pendingKeepAliveMode = nil
}
} message: {
Text("settings.flow.keepAlive.activeSession.message")
}
}
private var flowSkipAppSwitchLabel: some View {
VStack(alignment: .leading, spacing: Spacing.xxs) {
Text("settings.flow.skipAppSwitch.title")
.font(TypeStyle.body)
.foregroundStyle(palette.textPrimary)
Text("settings.flow.skipAppSwitch.subtitle")
.font(TypeStyle.caption2)
.foregroundStyle(palette.textTertiary)
}
}
private func applyKeepAliveModeChange(_ newMode: FlowKeepAliveMode) {
guard newMode != config.flowKeepAliveMode else { return }
if FlowSessionBridge.isSessionActive() {
pendingKeepAliveMode = newMode
showActiveFlowSessionAlert = true
return
}
config.flowKeepAliveMode = newMode
}
// MARK: - Engine
@@ -520,6 +564,31 @@ private struct AppearancePickerRow: View {
}
}
// MARK: - Flow keep-alive mode picker row
private struct FlowKeepAliveModePickerRow: View {
@Binding var selection: FlowKeepAliveMode
private var options: [(id: String, label: String)] {
FlowKeepAliveMode.allCases.map { mode in
(mode.rawValue, AppL10n.string(mode.labelKey))
}
}
var body: some View {
PickerRow(
title: AppL10n.string("settings.flow.keepAlive.title"),
options: options,
selection: Binding(
get: { selection.rawValue },
set: { newValue in
selection = FlowKeepAliveMode(rawValue: newValue) ?? .liveActivity
}
)
)
}
}
// MARK: - Flow inactivity picker row
private struct FlowInactivityPickerRow: View {
OSGKeyboard/en.lproj/Localizable.strings
+9
View File
@@ -418,6 +418,14 @@
/* Flow session policy */
"settings.flow.title" = "Voice session";
"settings.flow.keepAlive.title" = "Keep-alive mode";
"settings.flow.keepAlive.liveActivity" = "Dynamic Island";
"settings.flow.keepAlive.liveActivity.subtitle" = "Continuous mic session with inactivity timeout.";
"settings.flow.keepAlive.pictureInPicture" = "Picture in Picture";
"settings.flow.keepAlive.pictureInPicture.subtitle" = "Waveform PiP keeps the app alive; mic is released between utterances.";
"settings.flow.keepAlive.pictureInPicture.note" = "Picture in Picture stays active until you close it. Skip app switch is always on in this mode.";
"settings.flow.keepAlive.activeSession.title" = "End the current session first";
"settings.flow.keepAlive.activeSession.message" = "Stop the active voice session before changing keep-alive mode.";
"settings.flow.skipAppSwitch.title" = "Skip app switch";
"settings.flow.skipAppSwitch.subtitle" = "After a cold start, try to return to the app you came from.";
"settings.flow.inactivity.title" = "End session after inactivity";
@@ -438,6 +446,7 @@
"flow.coldStart.permission.title" = "Permission required";
"flow.coldStart.audio.title" = "Voice could not start";
"flow.coldStart.error.audioTimeout" = "The microphone did not become ready in time. It may be busy in another app. Please try again.";
"flow.pip.error.unavailable" = "Picture in Picture could not start. Check that PiP is allowed for OSGKeyboard in Settings.";
"flow.coldStart.action.settings" = "Open Settings";
"flow.coldStart.action.retry" = "Try Again";
"flow.coldStart.swipeHint" = "Swipe right along the bottom bar\nto return to your previous app.";
OSGKeyboard/zh-Hans.lproj/Localizable.strings
+9
View File
@@ -417,6 +417,14 @@
/* Flow 会话策略 */
"settings.flow.title" = "语音会话";
"settings.flow.keepAlive.title" = "保活方式";
"settings.flow.keepAlive.liveActivity" = "灵动岛";
"settings.flow.keepAlive.liveActivity.subtitle" = "麦克风常驻，可按无活动时长结束会话。";
"settings.flow.keepAlive.pictureInPicture" = "画中画";
"settings.flow.keepAlive.pictureInPicture.subtitle" = "波形画中画保活；句间释放麦克风。";
"settings.flow.keepAlive.pictureInPicture.note" = "画中画将持续保活，直到你关闭小窗。此模式下始终跳过应用切换。";
"settings.flow.keepAlive.activeSession.title" = "请先结束当前会话";
"settings.flow.keepAlive.activeSession.message" = "更改保活方式前，请先结束正在进行的语音会话。";
"settings.flow.skipAppSwitch.title" = "跳过应用切换";
"settings.flow.skipAppSwitch.subtitle" = "冷启动完成后，尝试自动返回来源 App。";
"settings.flow.inactivity.title" = "无活动后结束会话";
@@ -437,6 +445,7 @@
"flow.coldStart.permission.title" = "需要权限";
"flow.coldStart.audio.title" = "语音暂时无法启动";
"flow.coldStart.error.audioTimeout" = "麦克风未能及时就绪，可能正被通话或其他 App 占用。请稍后重试。";
"flow.pip.error.unavailable" = "无法启动画中画，请在系统设置中允许 OSGKeyboard 使用画中画。";
"flow.coldStart.action.settings" = "前往设置";
"flow.coldStart.action.retry" = "重试";
"flow.coldStart.swipeHint" = "沿屏幕底部的横条，从左向右滑动\n返回上一个 App。";