feat(flow): add Picture in Picture keep-alive mode

- Add FlowKeepAliveMode (Dynamic Island vs PiP) with iCloud sync
- Settings: keep-alive picker; inactivity timeout only for Live Activity
- PiP: waveform sample-buffer controller, mic released between utterances
- PiP sessions have no idle expiry; user closing PiP ends the session
- FlowSessionManager branches hostReady, session start, and utterance paths
- UIBackgroundModes picture-in-picture; bilingual strings and tests

Co-authored-by: Rocky <hkgood@users.noreply.github.com>
This commit is contained in:
Cursor Agent
2026-07-26 10:34:52 +00:00
parent e6f99d2744
commit 70ba3a4359
18 changed files with 801 additions and 36 deletions
+1
View File
@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
### Added ### Added
- **PiP Flow keep-alive**: Settings → Voice session lets you choose **Dynamic Island** (default, unchanged behaviour) or **Picture in Picture** — a live waveform PiP keeps the host alive with the mic released between utterances; closing PiP ends the session. / **PiP Flow 保活**:设置 → 语音会话可选 **灵动岛**(默认,行为不变)或 **画中画** — 实时波形 PiP 保活、句间释麦;关闭 PiP 即结束会话。
- **Mac MLX streaming ASR**: local dictation uses Qwen3-ASR via mlx-audio-swift with overlay partial preview, tail drain, vocabulary prompt, and polish-before-insert. / **Mac MLX 流式 ASR**:本地听写改用 mlx-audio-swift 的 Qwen3-ASR,支持浮层 partial 预览、尾部截断、词库 prompt 与润色后再插入。 - **Mac MLX streaming ASR**: local dictation uses Qwen3-ASR via mlx-audio-swift with overlay partial preview, tail drain, vocabulary prompt, and polish-before-insert. / **Mac MLX 流式 ASR**:本地听写改用 mlx-audio-swift 的 Qwen3-ASR,支持浮层 partial 预览、尾部截断、词库 prompt 与润色后再插入。
### Changed ### Changed
@@ -0,0 +1,307 @@
// FlowPictureInPictureController.swift
// OSGKeyboard · Main App
//
// PiP keep-alive for Flow sessions: enqueues live waveform sample buffers
// so the host process stays eligible for multitasking while the mic is off
// between utterances.
import AVFoundation
import AVKit
import CoreMedia
import UIKit
@MainActor
final class FlowPictureInPictureController: NSObject {
/// User closed the PiP window host should end the Flow session.
var onUserDismissed: (() -> Void)?
private(set) var isPictureInPictureActive = false
let displayLayer = AVSampleBufferDisplayLayer()
private var pipController: AVPictureInPictureController?
private var displayLink: CADisplayLink?
private weak var hostView: UIView?
private var waveformLevels: [Float] = Array(repeating: 0, count: 24)
private var isStoppingProgrammatically = false
private var frameIndex: Int64 = 0
// MARK: - Host view
func attachHostView(_ view: UIView) {
hostView = view
displayLayer.frame = view.bounds
displayLayer.videoGravity = .resizeAspectFill
displayLayer.removeFromSuperlayer()
view.layer.addSublayer(displayLayer)
configureControllerIfNeeded()
}
func updateHostLayoutIfNeeded() {
guard let hostView else { return }
displayLayer.frame = hostView.bounds
}
// MARK: - Lifecycle
@discardableResult
func start() -> Bool {
guard AVPictureInPictureController.isPictureInPictureSupported() else {
return false
}
configureControllerIfNeeded()
startFramePump()
guard pipController != nil else { return false }
if pipController?.isPictureInPictureActive == true {
isPictureInPictureActive = true
return true
}
enqueueWaveformFrame()
pipController?.startPictureInPicture()
return true
}
func startAndWait(timeout: TimeInterval = 4) async -> Bool {
if isPictureInPictureActive { return true }
guard start() else { return false }
let deadline = Date().addingTimeInterval(timeout)
while Date() < deadline {
if isPictureInPictureActive { return true }
if pipController?.isPictureInPictureActive == true {
isPictureInPictureActive = true
return true
}
try? await Task.sleep(nanoseconds: 50_000_000)
}
return isPictureInPictureActive
}
func stop() {
isStoppingProgrammatically = true
stopFramePump()
pipController?.stopPictureInPicture()
displayLayer.flushAndRemoveImage()
isPictureInPictureActive = false
isStoppingProgrammatically = false
}
func updateWaveformLevels(_ levels: [Float]) {
guard !levels.isEmpty else { return }
waveformLevels = levels
}
// MARK: - Private
private func configureControllerIfNeeded() {
guard pipController == nil else { return }
guard AVPictureInPictureController.isPictureInPictureSupported() else { return }
let contentSource = AVPictureInPictureController.ContentSource(
sampleBufferDisplayLayer: displayLayer,
playbackDelegate: self
)
let controller = AVPictureInPictureController(contentSource: contentSource)
controller.delegate = self
controller.canStartPictureInPictureAutomaticallyFromInline = true
pipController = controller
}
private func startFramePump() {
guard displayLink == nil else { return }
let link = CADisplayLink(target: self, selector: #selector(handleDisplayLink(_:)))
link.preferredFrameRateRange = CAFrameRateRange(minimum: 20, maximum: 30, preferred: 24)
link.add(to: .main, forMode: .common)
displayLink = link
}
private func stopFramePump() {
displayLink?.invalidate()
displayLink = nil
}
@objc private func handleDisplayLink(_ link: CADisplayLink) {
enqueueWaveformFrame()
updateHostLayoutIfNeeded()
if let pipController, !pipController.isPictureInPictureActive, pipController.isPictureInPicturePossible {
pipController.startPictureInPicture()
}
}
private func enqueueWaveformFrame() {
guard let sampleBuffer = makeWaveformSampleBuffer(levels: resolvedLevels()) else { return }
if displayLayer.status == .failed {
displayLayer.flush()
}
displayLayer.enqueue(sampleBuffer)
}
private func resolvedLevels() -> [Float] {
if waveformLevels.contains(where: { $0 > 0.02 }) {
return waveformLevels
}
// Idle breathing animation between utterances.
let phase = Float(frameIndex) * 0.12
return (0..<waveformLevels.count).map { index in
let wave = sin(phase + Float(index) * 0.45)
return max(0.04, 0.04 + wave * 0.03)
}
}
private func makeWaveformSampleBuffer(levels: [Float]) -> CMSampleBuffer? {
let width = 320
let height = 180
frameIndex += 1
var pixelBuffer: CVPixelBuffer?
let attrs: [String: Any] = [
kCVPixelBufferCGImageCompatibilityKey as String: true,
kCVPixelBufferCGBitmapContextCompatibilityKey as String: true,
]
let status = CVPixelBufferCreate(
kCFAllocatorDefault,
width,
height,
kCVPixelFormatType_32BGRA,
attrs as CFDictionary,
&pixelBuffer
)
guard status == kCVReturnSuccess, let pixelBuffer else { return nil }
CVPixelBufferLockBaseAddress(pixelBuffer, [])
defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) }
guard let base = CVPixelBufferGetBaseAddress(pixelBuffer) else { return nil }
let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
let colorSpace = CGColorSpaceCreateDeviceRGB()
guard let context = CGContext(
data: base,
width: width,
height: height,
bitsPerComponent: 8,
bytesPerRow: bytesPerRow,
space: colorSpace,
bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue
) else { return nil }
// Dark backdrop + accent waveform bars.
context.setFillColor(UIColor(red: 0.07, green: 0.09, blue: 0.11, alpha: 1).cgColor)
context.fill(CGRect(x: 0, y: 0, width: width, height: height))
let barCount = max(levels.count, 1)
let barWidth = CGFloat(width) / CGFloat(barCount * 2)
let accent = UIColor(red: 0.20, green: 0.78, blue: 0.55, alpha: 1)
context.setFillColor(accent.cgColor)
for (index, level) in levels.enumerated() {
let clamped = CGFloat(min(max(level, 0), 1))
let barHeight = max(6, clamped * CGFloat(height) * 0.72)
let x = (CGFloat(index) * 2 + 0.5) * barWidth
let rect = CGRect(
x: x,
y: (CGFloat(height) - barHeight) / 2,
width: barWidth,
height: barHeight
)
let path = UIBezierPath(roundedRect: rect, cornerRadius: barWidth * 0.35)
context.addPath(path.cgPath)
context.fillPath()
}
var formatDescription: CMFormatDescription?
CMVideoFormatDescriptionCreateForImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: pixelBuffer,
formatDescriptionOut: &formatDescription
)
guard let formatDescription else { return nil }
var timing = CMSampleTimingInfo(
duration: CMTime(value: 1, timescale: 24),
presentationTimeStamp: CMTime(value: frameIndex, timescale: 24),
decodeTimeStamp: .invalid
)
var sampleBuffer: CMSampleBuffer?
CMSampleBufferCreateForImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: pixelBuffer,
dataReady: true,
makeDataReadyCallback: nil,
refcon: nil,
formatDescription: formatDescription,
sampleTiming: &timing,
sampleBufferOut: &sampleBuffer
)
return sampleBuffer
}
}
// MARK: - AVPictureInPictureControllerDelegate
extension FlowPictureInPictureController: AVPictureInPictureControllerDelegate {
func pictureInPictureControllerDidStartPictureInPicture(
_ pictureInPictureController: AVPictureInPictureController
) {
isPictureInPictureActive = true
}
func pictureInPictureControllerDidStopPictureInPicture(
_ pictureInPictureController: AVPictureInPictureController
) {
isPictureInPictureActive = false
stopFramePump()
guard !isStoppingProgrammatically else { return }
onUserDismissed?()
}
func pictureInPictureController(
_ pictureInPictureController: AVPictureInPictureController,
restoreUserInterfaceForPictureInPictureStopWithCompletionHandler completionHandler: @escaping (Bool) -> Void
) {
completionHandler(true)
}
}
// MARK: - AVPictureInPictureSampleBufferPlaybackDelegate
extension FlowPictureInPictureController: AVPictureInPictureSampleBufferPlaybackDelegate {
func pictureInPictureController(
_ pictureInPictureController: AVPictureInPictureController,
setPlaying playing: Bool
) {
if playing {
startFramePump()
} else {
stopFramePump()
}
}
func pictureInPictureControllerTimeRangeForPlayback(
_ pictureInPictureController: AVPictureInPictureController
) -> CMTimeRange {
CMTimeRange(start: .zero, duration: CMTime(value: 3600, timescale: 1))
}
func pictureInPictureControllerIsPlaybackPaused(
_ pictureInPictureController: AVPictureInPictureController
) -> Bool {
false
}
func pictureInPictureController(
_ pictureInPictureController: AVPictureInPictureController,
didTransitionToRenderSize newRenderSize: CMVideoDimensions
) {}
func pictureInPictureController(
_ pictureInPictureController: AVPictureInPictureController,
skipByInterval skipInterval: CMTime,
completion completionHandler: @escaping () -> Void
) {
completionHandler()
}
}
+173 -7
View File
@@ -23,6 +23,7 @@ final class FlowSessionManager: ObservableObject {
@Published var coldStartContext: FlowColdStartContext? @Published var coldStartContext: FlowColdStartContext?
private let capture = FlowContinuousCapture() private let capture = FlowContinuousCapture()
private let pipController = FlowPictureInPictureController()
private let store = AppGroupStore() private let store = AppGroupStore()
/// Cloud-engine polish; local engine runs through built-in DeepSeek polish. /// Cloud-engine polish; local engine runs through built-in DeepSeek polish.
private let polisher = PolishingService() private let polisher = PolishingService()
@@ -78,6 +79,14 @@ final class FlowSessionManager: ObservableObject {
private var coldStartRecoveryTask: Task<Void, Never>? private var coldStartRecoveryTask: Task<Void, Never>?
/// Initial proof window cold mic sessions often need >2.5s after app switch. /// Initial proof window cold mic sessions often need >2.5s after app switch.
private static let coldStartAudioProofTimeout: TimeInterval = 6 private static let coldStartAudioProofTimeout: TimeInterval = 6
private var usesPiPKeepAlive: Bool {
FlowSessionPolicy.keepAliveMode() == .pictureInPicture
}
func attachPiPHostView(_ view: UIView) {
pipController.attachHostView(view)
}
/// Guards the once-per-process launch reconciliation (scene reconnects /// Guards the once-per-process launch reconciliation (scene reconnects
/// recreate the `@StateObject`-owned manager within the same process). /// recreate the `@StateObject`-owned manager within the same process).
private static var didRunLaunchReconciliation = false private static var didRunLaunchReconciliation = false
@@ -122,6 +131,11 @@ final class FlowSessionManager: ObservableObject {
kind: .recognitionInterrupted kind: .recognitionInterrupted
) )
} }
pipController.onUserDismissed = { [weak self] in
guard let self, self.isActive else { return }
self.debug("PiP dismissed by user — ending Flow session")
self.endSession()
}
FlowTerminationCoordinator.register(self) FlowTerminationCoordinator.register(self)
} }
@@ -276,7 +290,9 @@ final class FlowSessionManager: ObservableObject {
// hit the same audio-proof timeout. // hit the same audio-proof timeout.
coldStartRecoveryTask?.cancel() coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = nil coldStartRecoveryTask = nil
if capture.running { if usesPiPKeepAlive {
pipController.stop()
} else if capture.running {
capture.stop() capture.stop()
} }
sessionASR?.cancel() sessionASR?.cancel()
@@ -330,6 +346,7 @@ final class FlowSessionManager: ObservableObject {
if capture.running { if capture.running {
capture.stop() capture.stop()
} }
pipController.stop()
endBackgroundKeepAlive() endBackgroundKeepAlive()
ScreenWakeLock.release() ScreenWakeLock.release()
@@ -400,6 +417,7 @@ final class FlowSessionManager: ObservableObject {
isUtteranceProcessing = false isUtteranceProcessing = false
capture.stop() capture.stop()
pipController.stop()
endBackgroundKeepAlive() endBackgroundKeepAlive()
ScreenWakeLock.release() ScreenWakeLock.release()
sessionASR = nil sessionASR = nil
@@ -416,6 +434,10 @@ final class FlowSessionManager: ObservableObject {
} }
func extendSession(duration: TimeInterval? = nil) { func extendSession(duration: TimeInterval? = nil) {
guard !usesPiPKeepAlive else {
refreshHostReady()
return
}
let resolved = duration ?? FlowSessionPolicy.sessionDuration() let resolved = duration ?? FlowSessionPolicy.sessionDuration()
FlowSessionBridge.extendSession(by: resolved) FlowSessionBridge.extendSession(by: resolved)
sessionExpiresAt = Date().addingTimeInterval(resolved) sessionExpiresAt = Date().addingTimeInterval(resolved)
@@ -489,6 +511,10 @@ final class FlowSessionManager: ObservableObject {
private func reactivateCaptureIfNeeded() async { private func reactivateCaptureIfNeeded() async {
guard isActive else { return } guard isActive else { return }
if usesPiPKeepAlive, !isUtteranceRecording, !isUtteranceProcessing, !capture.running {
refreshHostReady()
return
}
// A system interruption (call / Siri) may be in progress. Probe it: // A system interruption (call / Siri) may be in progress. Probe it:
// `setActive(true)` inside `reassertIfRunning` fails while the // `setActive(true)` inside `reassertIfRunning` fails while the
// interruption is live and succeeds once it ends which also covers // interruption is live and succeeds once it ends which also covers
@@ -559,12 +585,22 @@ final class FlowSessionManager: ObservableObject {
let pollingAlive = pollingTask != nil && pollingTask?.isCancelled != true let pollingAlive = pollingTask != nil && pollingTask?.isCancelled != true
let hasRecentAudio = capture.engineHasRecentAudio(maxAge: 2) let hasRecentAudio = capture.engineHasRecentAudio(maxAge: 2)
let canAcceptUtterance = capture.engineIsLive let canAcceptUtterance: Bool
if usesPiPKeepAlive {
canAcceptUtterance = pipController.isPictureInPictureActive
&& pollingAlive
&& !isUtteranceRecording
&& !isUtteranceProcessing
&& sessionWarning == nil
&& !capture.isInterrupted
} else {
canAcceptUtterance = capture.engineIsLive
&& pollingAlive && pollingAlive
&& hasRecentAudio && hasRecentAudio
&& !isUtteranceRecording && !isUtteranceRecording
&& !isUtteranceProcessing && !isUtteranceProcessing
&& sessionWarning == nil && sessionWarning == nil
}
let reason: FlowReadySnapshot.Reason let reason: FlowReadySnapshot.Reason
if canAcceptUtterance { if canAcceptUtterance {
@@ -575,9 +611,11 @@ final class FlowSessionManager: ObservableObject {
reason = .recording reason = .recording
} else if isUtteranceProcessing { } else if isUtteranceProcessing {
reason = .processing reason = .processing
} else if !capture.engineIsLive { } else if usesPiPKeepAlive, !pipController.isPictureInPictureActive {
reason = .starting
} else if !usesPiPKeepAlive, !capture.engineIsLive {
reason = .audioEngineNotLive reason = .audioEngineNotLive
} else if !hasRecentAudio { } else if !usesPiPKeepAlive, !hasRecentAudio {
reason = .waitingForAudioProof reason = .waitingForAudioProof
} else { } else {
reason = .starting reason = .starting
@@ -650,6 +688,11 @@ final class FlowSessionManager: ObservableObject {
/// custom keyboard extension sees green immediately. /// custom keyboard extension sees green immediately.
func refreshForInlineKeyboardFocus() async { func refreshForInlineKeyboardFocus() async {
guard isActive else { return } guard isActive else { return }
if usesPiPKeepAlive {
refreshHostReady()
FlowSessionBridge.writeHeartbeat()
return
}
await reactivateCaptureIfNeeded() await reactivateCaptureIfNeeded()
refreshHostReady() refreshHostReady()
if !FlowSessionBridge.isHostReady() { if !FlowSessionBridge.isHostReady() {
@@ -662,7 +705,7 @@ final class FlowSessionManager: ObservableObject {
/// Extend expiry after utterance completion based on the inactivity policy. /// Extend expiry after utterance completion based on the inactivity policy.
private func touchSessionActivity() { private func touchSessionActivity() {
guard isActive else { return } guard isActive, !usesPiPKeepAlive else { return }
FlowSessionBridge.touchLastActivity() FlowSessionBridge.touchLastActivity()
if let expires = FlowSessionBridge.sessionExpiresAt() { if let expires = FlowSessionBridge.sessionExpiresAt() {
sessionExpiresAt = Date(timeIntervalSince1970: expires) sessionExpiresAt = Date(timeIntervalSince1970: expires)
@@ -693,6 +736,25 @@ final class FlowSessionManager: ObservableObject {
return return
} }
if usesPiPKeepAlive {
let pipReady = await pipController.startAndWait()
guard pipReady else {
let message = AppL10n.string("flow.pip.error.unavailable")
sessionWarning = message
traceState("startSessionAsync.failed", extra: "reason=pipUnavailable")
FlowSessionBridge.setHostReady(false)
if isColdStartHandoff {
showColdStartAudioFailure(message: message)
}
debug("PiP keep-alive failed to start")
return
}
activateFlowSessionAfterPiPProof(duration: duration)
traceState("startSessionAsync.ready")
debug("Flow session started (PiP keep-alive), mic released between utterances")
return
}
do { do {
try capture.start() try capture.start()
} catch { } catch {
@@ -729,6 +791,31 @@ final class FlowSessionManager: ObservableObject {
debug("Flow session started (\(Int(duration ?? FlowSessionPolicy.sessionDuration()))s inactivity window), continuous capture running") debug("Flow session started (\(Int(duration ?? FlowSessionPolicy.sessionDuration()))s inactivity window), continuous capture running")
} }
private func activateFlowSessionAfterPiPProof(duration: TimeInterval?) {
let sessionId = activeSessionId ?? UUID()
activeSessionId = sessionId
lastHandledCommandSeq = 0
FlowSessionBridge.markSessionActive(duration: duration, sessionId: sessionId)
FlowSessionDarwin.postSessionChanged()
isActive = true
ScreenWakeLock.acquire()
sessionExpiresAt = nil
startHeartbeat()
startCommandObserver()
startPolling()
startLevelPublishing()
expiryTask?.cancel()
expiryTask = nil
bindSessionASRIfNeeded()
scheduleASRWarmup()
FlowLiveActivityController.startSession()
refreshHostReady()
traceState("activateFlowSessionAfterPiPProof.done")
}
private func activateFlowSessionAfterAudioProof(duration: TimeInterval?) { private func activateFlowSessionAfterAudioProof(duration: TimeInterval?) {
let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration() let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration()
let sessionId = activeSessionId ?? UUID() let sessionId = activeSessionId ?? UUID()
@@ -756,6 +843,12 @@ final class FlowSessionManager: ObservableObject {
private func prepareExistingSessionForColdStartReturn() async { private func prepareExistingSessionForColdStartReturn() async {
guard isColdStartHandoff, isActive else { return } guard isColdStartHandoff, isActive else { return }
if usesPiPKeepAlive {
sessionWarning = nil
refreshHostReady()
handleColdStartAfterSessionReady()
return
}
await reactivateCaptureIfNeeded() await reactivateCaptureIfNeeded()
guard await waitForAudioProof() else { guard await waitForAudioProof() else {
let message = AppL10n.string("flow.coldStart.error.audioTimeout") let message = AppL10n.string("flow.coldStart.error.audioTimeout")
@@ -809,7 +902,7 @@ final class FlowSessionManager: ObservableObject {
} }
private func scheduleAutoReturnToHostIfNeeded(hostEntry: HostAppEntry?) { private func scheduleAutoReturnToHostIfNeeded(hostEntry: HostAppEntry?) {
let skipSwitch = FlowSessionPolicy.skipAppSwitch() let skipSwitch = usesPiPKeepAlive || FlowSessionPolicy.skipAppSwitch()
guard skipSwitch, hostEntry != nil else { return } guard skipSwitch, hostEntry != nil else { return }
Task { @MainActor [weak self] in Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: 450_000_000) try? await Task.sleep(nanoseconds: 450_000_000)
@@ -829,6 +922,21 @@ final class FlowSessionManager: ObservableObject {
coldStartRecoveryTask?.cancel() coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = Task { @MainActor [weak self] in coldStartRecoveryTask = Task { @MainActor [weak self] in
guard let self else { return } guard let self else { return }
if self.usesPiPKeepAlive {
let recovered = await self.pipController.startAndWait()
self.traceState("coldStartRecovery.pip", extra: "recovered=\(recovered)")
guard !Task.isCancelled, self.isColdStartHandoff else { return }
if recovered {
if self.isActive {
self.refreshHostReady()
self.handleColdStartAfterSessionReady()
} else {
self.activateFlowSessionAfterPiPProof(duration: duration)
self.handleColdStartAfterSessionReady()
}
}
return
}
var recovered = false var recovered = false
for attempt in 1...3 { for attempt in 1...3 {
guard !Task.isCancelled, self.isColdStartHandoff else { return } guard !Task.isCancelled, self.isColdStartHandoff else { return }
@@ -1000,7 +1108,12 @@ final class FlowSessionManager: ObservableObject {
switch command.action { switch command.action {
case .startRecording: case .startRecording:
guard !isUtteranceRecording, !isUtteranceProcessing else { return } guard !isUtteranceRecording, !isUtteranceProcessing else { return }
beginUtterance(utteranceId: command.utteranceId, commandSeq: command.commandSeq) Task { @MainActor [weak self] in
await self?.handleStartRecordingCommand(
utteranceId: command.utteranceId,
commandSeq: command.commandSeq
)
}
case .stopRecording: case .stopRecording:
guard currentUtteranceId == command.utteranceId else { return } guard currentUtteranceId == command.utteranceId else { return }
if isUtteranceRecording { if isUtteranceRecording {
@@ -1070,6 +1183,42 @@ final class FlowSessionManager: ObservableObject {
) )
} }
private func handleStartRecordingCommand(utteranceId: UUID?, commandSeq: Int64) async {
if usesPiPKeepAlive {
refreshHostReady()
let micReady = await ensureCaptureReadyForPiPUtterance()
guard micReady else {
failUtterance(
message: AppL10n.string("flow.coldStart.error.audioTimeout"),
kind: .audioUnavailable
)
return
}
}
beginUtterance(utteranceId: utteranceId, commandSeq: commandSeq)
}
private func ensureCaptureReadyForPiPUtterance() async -> Bool {
if capture.engineHasRecentAudio(maxAge: 2) {
return true
}
do {
try capture.start()
} catch {
debug("PiP utterance capture start failed: \(error.localizedDescription)")
return false
}
return await capture.awaitAudioFlowing(timeout: Self.coldStartAudioProofTimeout)
}
private func releaseCaptureAfterPiPUtteranceIfNeeded() {
guard usesPiPKeepAlive, capture.running else { return }
guard !isUtteranceRecording, !isUtteranceProcessing else { return }
capture.stop()
pipController.updateWaveformLevels([])
refreshHostReady()
}
private func beginUtterance(utteranceId: UUID? = nil, commandSeq: Int64 = 0) { private func beginUtterance(utteranceId: UUID? = nil, commandSeq: Int64 = 0) {
guard capture.engineHasRecentAudio(maxAge: 2) else { guard capture.engineHasRecentAudio(maxAge: 2) else {
traceState("beginUtterance.blocked", extra: "reason=audioNotRecent") traceState("beginUtterance.blocked", extra: "reason=audioNotRecent")
@@ -1207,6 +1356,10 @@ final class FlowSessionManager: ObservableObject {
guard let self else { return } guard let self else { return }
let drainReport = await self.capture.endUtteranceAndDrain() let drainReport = await self.capture.endUtteranceAndDrain()
FlowDiagnostics.logDrain(drainReport) FlowDiagnostics.logDrain(drainReport)
if self.usesPiPKeepAlive {
self.capture.stop()
self.pipController.updateWaveformLevels([])
}
await self.finalizeUtterance( await self.finalizeUtterance(
sessionId: drainingSessionId, sessionId: drainingSessionId,
utteranceId: drainingUtteranceId, utteranceId: drainingUtteranceId,
@@ -1229,6 +1382,7 @@ final class FlowSessionManager: ObservableObject {
chunkedPipeline = nil chunkedPipeline = nil
asr.cancel() asr.cancel()
capture.cancelUtterance() capture.cancelUtterance()
releaseCaptureAfterPiPUtteranceIfNeeded()
currentPartial = "" currentPartial = ""
lastFinal = "" lastFinal = ""
chunkWarnings = [] chunkWarnings = []
@@ -1255,6 +1409,7 @@ final class FlowSessionManager: ObservableObject {
chunkedPipeline = nil chunkedPipeline = nil
asr.cancel() asr.cancel()
capture.cancelUtterance() capture.cancelUtterance()
releaseCaptureAfterPiPUtteranceIfNeeded()
currentPartial = "" currentPartial = ""
lastFinal = "" lastFinal = ""
chunkWarnings = [] chunkWarnings = []
@@ -1277,6 +1432,8 @@ final class FlowSessionManager: ObservableObject {
finalizeTask?.cancel() finalizeTask?.cancel()
finalizeTask = nil finalizeTask = nil
chunkedPipeline = nil chunkedPipeline = nil
capture.cancelUtterance()
releaseCaptureAfterPiPUtteranceIfNeeded()
currentPartial = "" currentPartial = ""
lastFinal = "" lastFinal = ""
chunkWarnings = [] chunkWarnings = []
@@ -1589,6 +1746,9 @@ final class FlowSessionManager: ObservableObject {
while !Task.isCancelled { while !Task.isCancelled {
guard let self, self.isActive else { break } guard let self, self.isActive else { break }
let levels = self.capture.currentAudioLevels() let levels = self.capture.currentAudioLevels()
if self.usesPiPKeepAlive {
self.pipController.updateWaveformLevels(levels)
}
if levels.contains(where: { $0 > 0 }) { if levels.contains(where: { $0 > 0 }) {
FlowSessionBridge.storeAudioLevels(levels) FlowSessionBridge.storeAudioLevels(levels)
} }
@@ -1612,8 +1772,13 @@ final class FlowSessionManager: ObservableObject {
while !Task.isCancelled { while !Task.isCancelled {
guard let self else { break } guard let self else { break }
if self.isActive, !self.capture.engineIsLive { if self.isActive, !self.capture.engineIsLive {
let shouldReassert = !self.usesPiPKeepAlive
|| self.isUtteranceRecording
|| self.isUtteranceProcessing
if shouldReassert {
await self.reactivateCaptureIfNeeded() await self.reactivateCaptureIfNeeded()
} }
}
FlowSessionBridge.writeHeartbeat() FlowSessionBridge.writeHeartbeat()
self.refreshHostReady() self.refreshHostReady()
tick += 1 tick += 1
@@ -1627,6 +1792,7 @@ final class FlowSessionManager: ObservableObject {
} }
private func scheduleExpiry(after duration: TimeInterval) { private func scheduleExpiry(after duration: TimeInterval) {
guard !usesPiPKeepAlive else { return }
expiryTask?.cancel() expiryTask?.cancel()
expiryTask = Task { @MainActor [weak self] in expiryTask = Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: UInt64(duration * 1_000_000_000)) try? await Task.sleep(nanoseconds: UInt64(duration * 1_000_000_000))
+23
View File
@@ -0,0 +1,23 @@
// FlowPiPHostView.swift
// OSGKeyboard · Main App
//
// Hidden host for the PiP sample-buffer display layer (must live in the window hierarchy).
import SwiftUI
import UIKit
struct FlowPiPHostView: UIViewRepresentable {
let attach: (UIView) -> Void
func makeUIView(context: Context) -> UIView {
let view = UIView(frame: CGRect(x: 0, y: 0, width: 2, height: 2))
view.isUserInteractionEnabled = false
view.backgroundColor = .clear
attach(view)
return view
}
func updateUIView(_ uiView: UIView, context: Context) {
attach(uiView)
}
}
+8
View File
@@ -37,6 +37,14 @@ struct MainAppRoot: View {
} }
} }
.animation(.easeInOut(duration: 0.2), value: flowManager.coldStartContext != nil) .animation(.easeInOut(duration: 0.2), value: flowManager.coldStartContext != nil)
.background {
FlowPiPHostView { view in
flowManager.attachPiPHostView(view)
}
.frame(width: 2, height: 2)
.opacity(0.001)
.allowsHitTesting(false)
}
.onAppear { .onAppear {
flowManager.setAppForeground(scenePhase == .active) flowManager.setAppForeground(scenePhase == .active)
// Register the URL handler BEFORE the foreground auto-start. // Register the URL handler BEFORE the foreground auto-start.
+85 -16
View File
@@ -29,6 +29,8 @@ struct SettingsView: View {
// Dynamic locale list loaded from SFSpeechRecognizer on first appear. // Dynamic locale list loaded from SFSpeechRecognizer on first appear.
@State private var dynamicLocales: [(id: String, onDevice: Bool)] = [] @State private var dynamicLocales: [(id: String, onDevice: Bool)] = []
@State private var showResetConfirmation = false @State private var showResetConfirmation = false
@State private var showActiveFlowSessionAlert = false
@State private var pendingKeepAliveMode: FlowKeepAliveMode?
// v0.2.0: no on-device model manager / pending download state // v0.2.0: no on-device model manager / pending download state
// iOS `SpeechAnalyzer` ships with iOS 26 and needs nothing // iOS `SpeechAnalyzer` ships with iOS 26 and needs nothing
// downloaded. // downloaded.
@@ -105,7 +107,58 @@ struct SettingsView: View {
VStack(alignment: .leading, spacing: SettingsListMetrics.sectionLabelSpacing) { VStack(alignment: .leading, spacing: SettingsListMetrics.sectionLabelSpacing) {
sectionHeader("settings.flow.title") sectionHeader("settings.flow.title")
VStack(spacing: 0) { VStack(spacing: 0) {
FlowKeepAliveModePickerRow(
selection: Binding(
get: { config.flowKeepAliveMode },
set: { newMode in
applyKeepAliveModeChange(newMode)
}
)
)
if config.flowKeepAliveMode == .liveActivity {
Divider().background(palette.divider)
FlowInactivityPickerRow(
selection: Binding(
get: { config.flowInactivityDuration },
set: { config.flowInactivityDuration = $0 }
)
)
Divider().background(palette.divider)
Toggle(isOn: $config.flowSkipAppSwitch) { Toggle(isOn: $config.flowSkipAppSwitch) {
flowSkipAppSwitchLabel
}
.tint(palette.accent)
.settingsListRow()
} else {
Divider().background(palette.divider)
Text("settings.flow.keepAlive.pictureInPicture.note")
.font(TypeStyle.caption2)
.foregroundStyle(palette.textTertiary)
.frame(maxWidth: .infinity, alignment: .leading)
.settingsListRow()
}
}
.background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.xl, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: Radius.xl, style: .continuous)
.stroke(palette.divider, lineWidth: 0.5)
)
}
.alert("settings.flow.keepAlive.activeSession.title", isPresented: $showActiveFlowSessionAlert) {
Button("common.done", role: .cancel) {
pendingKeepAliveMode = nil
}
} message: {
Text("settings.flow.keepAlive.activeSession.message")
}
}
private var flowSkipAppSwitchLabel: some View {
VStack(alignment: .leading, spacing: Spacing.xxs) { VStack(alignment: .leading, spacing: Spacing.xxs) {
Text("settings.flow.skipAppSwitch.title") Text("settings.flow.skipAppSwitch.title")
.font(TypeStyle.body) .font(TypeStyle.body)
@@ -115,24 +168,15 @@ struct SettingsView: View {
.foregroundStyle(palette.textTertiary) .foregroundStyle(palette.textTertiary)
} }
} }
.tint(palette.accent)
.settingsListRow()
Divider().background(palette.divider) private func applyKeepAliveModeChange(_ newMode: FlowKeepAliveMode) {
guard newMode != config.flowKeepAliveMode else { return }
FlowInactivityPickerRow( if FlowSessionBridge.isSessionActive() {
selection: Binding( pendingKeepAliveMode = newMode
get: { config.flowInactivityDuration }, showActiveFlowSessionAlert = true
set: { config.flowInactivityDuration = $0 } return
)
)
}
.background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.xl, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: Radius.xl, style: .continuous)
.stroke(palette.divider, lineWidth: 0.5)
)
} }
config.flowKeepAliveMode = newMode
} }
// MARK: - Engine // MARK: - Engine
@@ -520,6 +564,31 @@ private struct AppearancePickerRow: View {
} }
} }
// MARK: - Flow keep-alive mode picker row
private struct FlowKeepAliveModePickerRow: View {
@Binding var selection: FlowKeepAliveMode
private var options: [(id: String, label: String)] {
FlowKeepAliveMode.allCases.map { mode in
(mode.rawValue, AppL10n.string(mode.labelKey))
}
}
var body: some View {
PickerRow(
title: AppL10n.string("settings.flow.keepAlive.title"),
options: options,
selection: Binding(
get: { selection.rawValue },
set: { newValue in
selection = FlowKeepAliveMode(rawValue: newValue) ?? .liveActivity
}
)
)
}
}
// MARK: - Flow inactivity picker row // MARK: - Flow inactivity picker row
private struct FlowInactivityPickerRow: View { private struct FlowInactivityPickerRow: View {
+9
View File
@@ -418,6 +418,14 @@
/* Flow session policy */ /* Flow session policy */
"settings.flow.title" = "Voice session"; "settings.flow.title" = "Voice session";
"settings.flow.keepAlive.title" = "Keep-alive mode";
"settings.flow.keepAlive.liveActivity" = "Dynamic Island";
"settings.flow.keepAlive.liveActivity.subtitle" = "Continuous mic session with inactivity timeout.";
"settings.flow.keepAlive.pictureInPicture" = "Picture in Picture";
"settings.flow.keepAlive.pictureInPicture.subtitle" = "Waveform PiP keeps the app alive; mic is released between utterances.";
"settings.flow.keepAlive.pictureInPicture.note" = "Picture in Picture stays active until you close it. Skip app switch is always on in this mode.";
"settings.flow.keepAlive.activeSession.title" = "End the current session first";
"settings.flow.keepAlive.activeSession.message" = "Stop the active voice session before changing keep-alive mode.";
"settings.flow.skipAppSwitch.title" = "Skip app switch"; "settings.flow.skipAppSwitch.title" = "Skip app switch";
"settings.flow.skipAppSwitch.subtitle" = "After a cold start, try to return to the app you came from."; "settings.flow.skipAppSwitch.subtitle" = "After a cold start, try to return to the app you came from.";
"settings.flow.inactivity.title" = "End session after inactivity"; "settings.flow.inactivity.title" = "End session after inactivity";
@@ -438,6 +446,7 @@
"flow.coldStart.permission.title" = "Permission required"; "flow.coldStart.permission.title" = "Permission required";
"flow.coldStart.audio.title" = "Voice could not start"; "flow.coldStart.audio.title" = "Voice could not start";
"flow.coldStart.error.audioTimeout" = "The microphone did not become ready in time. It may be busy in another app. Please try again."; "flow.coldStart.error.audioTimeout" = "The microphone did not become ready in time. It may be busy in another app. Please try again.";
"flow.pip.error.unavailable" = "Picture in Picture could not start. Check that PiP is allowed for OSGKeyboard in Settings.";
"flow.coldStart.action.settings" = "Open Settings"; "flow.coldStart.action.settings" = "Open Settings";
"flow.coldStart.action.retry" = "Try Again"; "flow.coldStart.action.retry" = "Try Again";
"flow.coldStart.swipeHint" = "Swipe right along the bottom bar\nto return to your previous app."; "flow.coldStart.swipeHint" = "Swipe right along the bottom bar\nto return to your previous app.";
@@ -417,6 +417,14 @@
/* Flow 会话策略 */ /* Flow 会话策略 */
"settings.flow.title" = "语音会话"; "settings.flow.title" = "语音会话";
"settings.flow.keepAlive.title" = "保活方式";
"settings.flow.keepAlive.liveActivity" = "灵动岛";
"settings.flow.keepAlive.liveActivity.subtitle" = "麦克风常驻,可按无活动时长结束会话。";
"settings.flow.keepAlive.pictureInPicture" = "画中画";
"settings.flow.keepAlive.pictureInPicture.subtitle" = "波形画中画保活;句间释放麦克风。";
"settings.flow.keepAlive.pictureInPicture.note" = "画中画将持续保活,直到你关闭小窗。此模式下始终跳过应用切换。";
"settings.flow.keepAlive.activeSession.title" = "请先结束当前会话";
"settings.flow.keepAlive.activeSession.message" = "更改保活方式前,请先结束正在进行的语音会话。";
"settings.flow.skipAppSwitch.title" = "跳过应用切换"; "settings.flow.skipAppSwitch.title" = "跳过应用切换";
"settings.flow.skipAppSwitch.subtitle" = "冷启动完成后,尝试自动返回来源 App。"; "settings.flow.skipAppSwitch.subtitle" = "冷启动完成后,尝试自动返回来源 App。";
"settings.flow.inactivity.title" = "无活动后结束会话"; "settings.flow.inactivity.title" = "无活动后结束会话";
@@ -437,6 +445,7 @@
"flow.coldStart.permission.title" = "需要权限"; "flow.coldStart.permission.title" = "需要权限";
"flow.coldStart.audio.title" = "语音暂时无法启动"; "flow.coldStart.audio.title" = "语音暂时无法启动";
"flow.coldStart.error.audioTimeout" = "麦克风未能及时就绪,可能正被通话或其他 App 占用。请稍后重试。"; "flow.coldStart.error.audioTimeout" = "麦克风未能及时就绪,可能正被通话或其他 App 占用。请稍后重试。";
"flow.pip.error.unavailable" = "无法启动画中画,请在系统设置中允许 OSGKeyboard 使用画中画。";
"flow.coldStart.action.settings" = "前往设置"; "flow.coldStart.action.settings" = "前往设置";
"flow.coldStart.action.retry" = "重试"; "flow.coldStart.action.retry" = "重试";
"flow.coldStart.swipeHint" = "沿屏幕底部的横条,从左向右滑动\n返回上一个 App。"; "flow.coldStart.swipeHint" = "沿屏幕底部的横条,从左向右滑动\n返回上一个 App。";
@@ -50,6 +50,8 @@ public struct AppGroupConfiguration: Sendable, Equatable {
public static let settingsCloudPayloadV2 = "config.settings.cloudPayload.v2" public static let settingsCloudPayloadV2 = "config.settings.cloudPayload.v2"
/// When true, the host app auto-returns to the source app after a cold-start handoff. /// When true, the host app auto-returns to the source app after a cold-start handoff.
public static let flowSkipAppSwitch = "config.flowSkipAppSwitch" public static let flowSkipAppSwitch = "config.flowSkipAppSwitch"
/// Raw `FlowKeepAliveMode` value; mutually exclusive PiP vs Live Activity path.
public static let flowKeepAliveMode = "config.flowKeepAliveMode"
/// Raw `FlowInactivityDuration` value; session expires after this idle window. /// Raw `FlowInactivityDuration` value; session expires after this idle window.
public static let flowInactivityDuration = "config.flowInactivityDuration" public static let flowInactivityDuration = "config.flowInactivityDuration"
/// One-shot: remap previous product defaults (30m / 10m) 5m. /// One-shot: remap previous product defaults (30m / 10m) 5m.
@@ -88,6 +90,8 @@ public struct AppGroupConfiguration: Sendable, Equatable {
public var settingsICloudSyncEnabled: Bool public var settingsICloudSyncEnabled: Bool
/// Auto-return to the host app after `startflow` cold start (default on). /// Auto-return to the host app after `startflow` cold start (default on).
public var flowSkipAppSwitch: Bool public var flowSkipAppSwitch: Bool
/// PiP vs Live Activity keep-alive strategy (mutually exclusive).
public var flowKeepAliveMode: FlowKeepAliveMode
/// Idle timeout before the Flow session ends; resets on each utterance. /// Idle timeout before the Flow session ends; resets on each utterance.
public var flowInactivityDuration: FlowInactivityDuration public var flowInactivityDuration: FlowInactivityDuration
/// Whether local `SpeechAnalyzer` should attach the prepared custom language model. /// Whether local `SpeechAnalyzer` should attach the prepared custom language model.
@@ -262,6 +266,9 @@ public struct AppGroupConfiguration: Sendable, Equatable {
} }
return defaults.bool(forKey: Keys.flowSkipAppSwitch) return defaults.bool(forKey: Keys.flowSkipAppSwitch)
}(), }(),
flowKeepAliveMode: FlowKeepAliveMode.fromStored(
defaults.string(forKey: Keys.flowKeepAliveMode)
),
flowInactivityDuration: FlowInactivityDuration.fromStored( flowInactivityDuration: FlowInactivityDuration.fromStored(
defaults.string(forKey: Keys.flowInactivityDuration) defaults.string(forKey: Keys.flowInactivityDuration)
), ),
@@ -370,6 +377,7 @@ public struct AppGroupConfiguration: Sendable, Equatable {
defaults.set(polishIntensity.rawValue, forKey: Keys.polishIntensity) defaults.set(polishIntensity.rawValue, forKey: Keys.polishIntensity)
defaults.set(llmThinkingEnabled, forKey: Keys.llmThinkingEnabled) defaults.set(llmThinkingEnabled, forKey: Keys.llmThinkingEnabled)
defaults.set(flowSkipAppSwitch, forKey: Keys.flowSkipAppSwitch) defaults.set(flowSkipAppSwitch, forKey: Keys.flowSkipAppSwitch)
defaults.set(flowKeepAliveMode.rawValue, forKey: Keys.flowKeepAliveMode)
defaults.set(flowInactivityDuration.rawValue, forKey: Keys.flowInactivityDuration) defaults.set(flowInactivityDuration.rawValue, forKey: Keys.flowInactivityDuration)
defaults.set(localASRCustomLanguageModelEnabled, forKey: Keys.localASRCustomLanguageModelEnabled) defaults.set(localASRCustomLanguageModelEnabled, forKey: Keys.localASRCustomLanguageModelEnabled)
defaults.set(personalDictionaryICloudSyncEnabled, forKey: Keys.personalDictionaryICloudSyncEnabled) defaults.set(personalDictionaryICloudSyncEnabled, forKey: Keys.personalDictionaryICloudSyncEnabled)
@@ -0,0 +1,39 @@
// FlowKeepAliveMode.swift
// OSGKeyboard · Shared
//
// User-selectable Flow session keep-alive strategy (mutually exclusive).
import Foundation
public enum FlowKeepAliveMode: String, CaseIterable, Identifiable, Sendable, Codable {
/// Continuous audio capture + Live Activity (current default behaviour).
case liveActivity = "liveActivity"
/// Picture-in-picture waveform keep-alive; mic released between utterances.
case pictureInPicture = "pictureInPicture"
public var id: String { rawValue }
/// Existing installs keep the Live Activity / continuous-capture path.
public static let `default`: FlowKeepAliveMode = .liveActivity
public var labelKey: String {
switch self {
case .liveActivity: return "settings.flow.keepAlive.liveActivity"
case .pictureInPicture: return "settings.flow.keepAlive.pictureInPicture"
}
}
public var subtitleKey: String {
switch self {
case .liveActivity: return "settings.flow.keepAlive.liveActivity.subtitle"
case .pictureInPicture: return "settings.flow.keepAlive.pictureInPicture.subtitle"
}
}
public static func fromStored(_ raw: String?) -> FlowKeepAliveMode {
guard let raw, let value = FlowKeepAliveMode(rawValue: raw) else {
return .default
}
return value
}
}
+18 -1
View File
@@ -237,7 +237,22 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
} }
} }
/// Idle window before an active Flow session expires; resets on each utterance. /// PiP vs Live Activity keep-alive (mutually exclusive).
@Published public var flowKeepAliveMode: FlowKeepAliveMode {
didSet {
guard !isApplyingConfiguration, flowKeepAliveMode != configuration.flowKeepAliveMode else { return }
configuration.flowKeepAliveMode = flowKeepAliveMode
if flowKeepAliveMode == .pictureInPicture {
configuration.flowSkipAppSwitch = true
if flowSkipAppSwitch != true {
flowSkipAppSwitch = true
}
}
persistConfiguration()
}
}
/// Idle window before an active Flow session expires; Live Activity mode only.
@Published public var flowInactivityDuration: FlowInactivityDuration { @Published public var flowInactivityDuration: FlowInactivityDuration {
didSet { didSet {
guard !isApplyingConfiguration, guard !isApplyingConfiguration,
@@ -347,6 +362,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
polishIntensity = configuration.polishIntensity polishIntensity = configuration.polishIntensity
llmThinkingEnabled = configuration.llmThinkingEnabled llmThinkingEnabled = configuration.llmThinkingEnabled
flowSkipAppSwitch = configuration.flowSkipAppSwitch flowSkipAppSwitch = configuration.flowSkipAppSwitch
flowKeepAliveMode = configuration.flowKeepAliveMode
flowInactivityDuration = configuration.flowInactivityDuration flowInactivityDuration = configuration.flowInactivityDuration
localASRCustomLanguageModelEnabled = configuration.localASRCustomLanguageModelEnabled localASRCustomLanguageModelEnabled = configuration.localASRCustomLanguageModelEnabled
isSyncingProviderAPIKey = true isSyncingProviderAPIKey = true
@@ -433,6 +449,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
polishIntensity = fresh.polishIntensity polishIntensity = fresh.polishIntensity
llmThinkingEnabled = fresh.llmThinkingEnabled llmThinkingEnabled = fresh.llmThinkingEnabled
flowSkipAppSwitch = fresh.flowSkipAppSwitch flowSkipAppSwitch = fresh.flowSkipAppSwitch
flowKeepAliveMode = fresh.flowKeepAliveMode
flowInactivityDuration = fresh.flowInactivityDuration flowInactivityDuration = fresh.flowInactivityDuration
localASRCustomLanguageModelEnabled = fresh.localASRCustomLanguageModelEnabled localASRCustomLanguageModelEnabled = fresh.localASRCustomLanguageModelEnabled
isSyncingProviderAPIKey = true isSyncingProviderAPIKey = true
@@ -29,6 +29,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
public var polishIntensity: SyncedField<PolishIntensity> public var polishIntensity: SyncedField<PolishIntensity>
public var llmThinkingEnabled: SyncedField<Bool> public var llmThinkingEnabled: SyncedField<Bool>
public var flowSkipAppSwitch: SyncedField<Bool> public var flowSkipAppSwitch: SyncedField<Bool>
public var flowKeepAliveMode: SyncedField<FlowKeepAliveMode>
public var flowInactivityDuration: SyncedField<FlowInactivityDuration> public var flowInactivityDuration: SyncedField<FlowInactivityDuration>
public init( public init(
@@ -50,6 +51,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
polishIntensity: SyncedField<PolishIntensity>, polishIntensity: SyncedField<PolishIntensity>,
llmThinkingEnabled: SyncedField<Bool>, llmThinkingEnabled: SyncedField<Bool>,
flowSkipAppSwitch: SyncedField<Bool>, flowSkipAppSwitch: SyncedField<Bool>,
flowKeepAliveMode: SyncedField<FlowKeepAliveMode>,
flowInactivityDuration: SyncedField<FlowInactivityDuration> flowInactivityDuration: SyncedField<FlowInactivityDuration>
) { ) {
self.schemaVersion = schemaVersion self.schemaVersion = schemaVersion
@@ -70,6 +72,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
self.polishIntensity = polishIntensity self.polishIntensity = polishIntensity
self.llmThinkingEnabled = llmThinkingEnabled self.llmThinkingEnabled = llmThinkingEnabled
self.flowSkipAppSwitch = flowSkipAppSwitch self.flowSkipAppSwitch = flowSkipAppSwitch
self.flowKeepAliveMode = flowKeepAliveMode
self.flowInactivityDuration = flowInactivityDuration self.flowInactivityDuration = flowInactivityDuration
} }
@@ -92,6 +95,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
case polishIntensity case polishIntensity
case llmThinkingEnabled case llmThinkingEnabled
case flowSkipAppSwitch case flowSkipAppSwitch
case flowKeepAliveMode
case flowInactivityDuration case flowInactivityDuration
} }
@@ -124,6 +128,14 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
forKey: .llmThinkingEnabled forKey: .llmThinkingEnabled
) ?? SyncedField(value: false, updatedAt: polishIntensity.updatedAt, deviceID: polishIntensity.deviceID) ) ?? SyncedField(value: false, updatedAt: polishIntensity.updatedAt, deviceID: polishIntensity.deviceID)
flowSkipAppSwitch = try container.decode(SyncedField<Bool>.self, forKey: .flowSkipAppSwitch) flowSkipAppSwitch = try container.decode(SyncedField<Bool>.self, forKey: .flowSkipAppSwitch)
flowKeepAliveMode = try container.decodeIfPresent(
SyncedField<FlowKeepAliveMode>.self,
forKey: .flowKeepAliveMode
) ?? SyncedField(
value: .liveActivity,
updatedAt: flowSkipAppSwitch.updatedAt,
deviceID: flowSkipAppSwitch.deviceID
)
flowInactivityDuration = try container.decode( flowInactivityDuration = try container.decode(
SyncedField<FlowInactivityDuration>.self, SyncedField<FlowInactivityDuration>.self,
forKey: .flowInactivityDuration forKey: .flowInactivityDuration
@@ -171,6 +183,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
polishIntensity.updatedAt, polishIntensity.updatedAt,
llmThinkingEnabled.updatedAt, llmThinkingEnabled.updatedAt,
flowSkipAppSwitch.updatedAt, flowSkipAppSwitch.updatedAt,
flowKeepAliveMode.updatedAt,
flowInactivityDuration.updatedAt, flowInactivityDuration.updatedAt,
].max() ?? .distantPast ].max() ?? .distantPast
} }
@@ -208,6 +221,7 @@ public extension SyncedAppSettingsV2 {
polishIntensity: field(configuration.polishIntensity), polishIntensity: field(configuration.polishIntensity),
llmThinkingEnabled: field(configuration.llmThinkingEnabled), llmThinkingEnabled: field(configuration.llmThinkingEnabled),
flowSkipAppSwitch: field(configuration.flowSkipAppSwitch), flowSkipAppSwitch: field(configuration.flowSkipAppSwitch),
flowKeepAliveMode: field(configuration.flowKeepAliveMode),
flowInactivityDuration: field(configuration.flowInactivityDuration) flowInactivityDuration: field(configuration.flowInactivityDuration)
) )
} }
@@ -237,6 +251,7 @@ public extension SyncedAppSettingsV2 {
polishIntensity: field(legacy.polishIntensity), polishIntensity: field(legacy.polishIntensity),
llmThinkingEnabled: field(false), llmThinkingEnabled: field(false),
flowSkipAppSwitch: field(legacy.flowSkipAppSwitch), flowSkipAppSwitch: field(legacy.flowSkipAppSwitch),
flowKeepAliveMode: field(.liveActivity),
flowInactivityDuration: field(legacy.flowInactivityDuration) flowInactivityDuration: field(legacy.flowInactivityDuration)
) )
} }
@@ -269,6 +284,7 @@ public extension SyncedAppSettingsV2 {
polishIntensity: .merge(local: local.polishIntensity, remote: remote.polishIntensity), polishIntensity: .merge(local: local.polishIntensity, remote: remote.polishIntensity),
llmThinkingEnabled: .merge(local: local.llmThinkingEnabled, remote: remote.llmThinkingEnabled), llmThinkingEnabled: .merge(local: local.llmThinkingEnabled, remote: remote.llmThinkingEnabled),
flowSkipAppSwitch: .merge(local: local.flowSkipAppSwitch, remote: remote.flowSkipAppSwitch), flowSkipAppSwitch: .merge(local: local.flowSkipAppSwitch, remote: remote.flowSkipAppSwitch),
flowKeepAliveMode: .merge(local: local.flowKeepAliveMode, remote: remote.flowKeepAliveMode),
flowInactivityDuration: .merge( flowInactivityDuration: .merge(
local: local.flowInactivityDuration, local: local.flowInactivityDuration,
remote: remote.flowInactivityDuration remote: remote.flowInactivityDuration
@@ -294,6 +310,7 @@ public extension SyncedAppSettingsV2 {
configuration.polishIntensity = polishIntensity.value configuration.polishIntensity = polishIntensity.value
configuration.llmThinkingEnabled = llmThinkingEnabled.value configuration.llmThinkingEnabled = llmThinkingEnabled.value
configuration.flowSkipAppSwitch = flowSkipAppSwitch.value configuration.flowSkipAppSwitch = flowSkipAppSwitch.value
configuration.flowKeepAliveMode = flowKeepAliveMode.value
configuration.flowInactivityDuration = flowInactivityDuration.value configuration.flowInactivityDuration = flowInactivityDuration.value
} }
@@ -321,6 +338,7 @@ public extension SyncedAppSettingsV2 {
patch(&copy.polishIntensity, value: configuration.polishIntensity) patch(&copy.polishIntensity, value: configuration.polishIntensity)
patch(&copy.llmThinkingEnabled, value: configuration.llmThinkingEnabled) patch(&copy.llmThinkingEnabled, value: configuration.llmThinkingEnabled)
patch(&copy.flowSkipAppSwitch, value: configuration.flowSkipAppSwitch) patch(&copy.flowSkipAppSwitch, value: configuration.flowSkipAppSwitch)
patch(&copy.flowKeepAliveMode, value: configuration.flowKeepAliveMode)
patch(&copy.flowInactivityDuration, value: configuration.flowInactivityDuration) patch(&copy.flowInactivityDuration, value: configuration.flowInactivityDuration)
return copy return copy
} }
@@ -351,6 +369,7 @@ public extension SyncedAppSettingsV2 {
touch(&copy.polishIntensity, value: configuration.polishIntensity) touch(&copy.polishIntensity, value: configuration.polishIntensity)
touch(&copy.llmThinkingEnabled, value: configuration.llmThinkingEnabled) touch(&copy.llmThinkingEnabled, value: configuration.llmThinkingEnabled)
touch(&copy.flowSkipAppSwitch, value: configuration.flowSkipAppSwitch) touch(&copy.flowSkipAppSwitch, value: configuration.flowSkipAppSwitch)
touch(&copy.flowKeepAliveMode, value: configuration.flowKeepAliveMode)
touch(&copy.flowInactivityDuration, value: configuration.flowInactivityDuration) touch(&copy.flowInactivityDuration, value: configuration.flowInactivityDuration)
return copy return copy
} }
@@ -312,11 +312,22 @@ public enum FlowSessionBridge {
defaults: UserDefaults? = nil defaults: UserDefaults? = nil
) { ) {
let store = resolvedDefaults(defaults) let store = resolvedDefaults(defaults)
let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration(defaults: store) if FlowSessionPolicy.usesInactivityExpiry(defaults: store) {
markSessionActiveWithExpiry(duration: duration, sessionId: sessionId, defaults: store)
} else {
markSessionActivePersistent(sessionId: sessionId, defaults: store)
}
}
/// PiP keep-alive: session stays valid until explicit teardown (no idle expiry).
public static func markSessionActivePersistent(
sessionId: UUID? = nil,
defaults: UserDefaults? = nil
) {
let store = resolvedDefaults(defaults)
let now = Date().timeIntervalSince1970 let now = Date().timeIntervalSince1970
let expires = now + resolvedDuration
store.set(true, forKey: FlowSessionKeys.flowSessionActive) store.set(true, forKey: FlowSessionKeys.flowSessionActive)
store.set(expires, forKey: FlowSessionKeys.flowSessionExpires) store.removeObject(forKey: FlowSessionKeys.flowSessionExpires)
store.set(now, forKey: FlowSessionKeys.lastActivityAt) store.set(now, forKey: FlowSessionKeys.lastActivityAt)
writeHeartbeat(defaults: store) writeHeartbeat(defaults: store)
clearTranscription(defaults: store) clearTranscription(defaults: store)
@@ -331,7 +342,7 @@ public enum FlowSessionBridge {
heartbeatAt: now, heartbeatAt: now,
engineMode: AppGroupConfiguration.load(fromAvailable: store).engineMode, engineMode: AppGroupConfiguration.load(fromAvailable: store).engineMode,
localeId: AppGroupConfiguration.load(fromAvailable: store).localeId, localeId: AppGroupConfiguration.load(fromAvailable: store).localeId,
sessionExpiresAt: expires, sessionExpiresAt: nil,
hostGeneration: store.string(forKey: FlowSessionKeys.hostGeneration) hostGeneration: store.string(forKey: FlowSessionKeys.hostGeneration)
) )
if let data = encode(snapshot) { if let data = encode(snapshot) {
@@ -343,6 +354,42 @@ public enum FlowSessionBridge {
flush(store) flush(store)
} }
private static func markSessionActiveWithExpiry(
duration: TimeInterval? = nil,
sessionId: UUID? = nil,
defaults: UserDefaults
) {
let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration(defaults: defaults)
let now = Date().timeIntervalSince1970
let expires = now + resolvedDuration
defaults.set(true, forKey: FlowSessionKeys.flowSessionActive)
defaults.set(expires, forKey: FlowSessionKeys.flowSessionExpires)
defaults.set(now, forKey: FlowSessionKeys.lastActivityAt)
writeHeartbeat(defaults: defaults)
clearTranscription(defaults: defaults)
defaults.removeObject(forKey: FlowSessionKeys.flowCommandPayload)
defaults.removeObject(forKey: FlowSessionKeys.flowResultPayload)
defaults.removeObject(forKey: FlowSessionKeys.flowAckPayload)
if let sessionId {
let snapshot = FlowReadySnapshot(
sessionId: sessionId,
ready: false,
reason: .starting,
heartbeatAt: now,
engineMode: AppGroupConfiguration.load(fromAvailable: defaults).engineMode,
localeId: AppGroupConfiguration.load(fromAvailable: defaults).localeId,
sessionExpiresAt: expires,
hostGeneration: defaults.string(forKey: FlowSessionKeys.hostGeneration)
)
if let data = encode(snapshot) {
defaults.set(data, forKey: FlowSessionKeys.flowReadyPayload)
}
} else {
defaults.removeObject(forKey: FlowSessionKeys.flowReadyPayload)
}
flush(defaults)
}
public static func markSessionInactive(defaults: UserDefaults? = nil) { public static func markSessionInactive(defaults: UserDefaults? = nil) {
let store = resolvedDefaults(defaults) let store = resolvedDefaults(defaults)
store.set(false, forKey: FlowSessionKeys.flowSessionActive) store.set(false, forKey: FlowSessionKeys.flowSessionActive)
@@ -372,6 +419,7 @@ public enum FlowSessionBridge {
defaults: UserDefaults? = nil defaults: UserDefaults? = nil
) { ) {
let store = resolvedDefaults(defaults) let store = resolvedDefaults(defaults)
guard FlowSessionPolicy.usesInactivityExpiry(defaults: store) else { return }
let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration(defaults: store) let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration(defaults: store)
let expires = Date().timeIntervalSince1970 + resolvedDuration let expires = Date().timeIntervalSince1970 + resolvedDuration
store.set(true, forKey: FlowSessionKeys.flowSessionActive) store.set(true, forKey: FlowSessionKeys.flowSessionActive)
@@ -382,6 +430,7 @@ public enum FlowSessionBridge {
/// Resets the inactivity timer after utterance completion or explicit activity. /// Resets the inactivity timer after utterance completion or explicit activity.
public static func touchLastActivity(defaults: UserDefaults? = nil) { public static func touchLastActivity(defaults: UserDefaults? = nil) {
let store = resolvedDefaults(defaults) let store = resolvedDefaults(defaults)
guard FlowSessionPolicy.usesInactivityExpiry(defaults: store) else { return }
let now = Date().timeIntervalSince1970 let now = Date().timeIntervalSince1970
let duration = FlowSessionPolicy.sessionDuration(defaults: store) let duration = FlowSessionPolicy.sessionDuration(defaults: store)
store.set(now, forKey: FlowSessionKeys.lastActivityAt) store.set(now, forKey: FlowSessionKeys.lastActivityAt)
@@ -419,6 +468,10 @@ public enum FlowSessionBridge {
let store = resolvedDefaults(defaults) let store = resolvedDefaults(defaults)
guard store.bool(forKey: FlowSessionKeys.flowSessionActive) else { return false } guard store.bool(forKey: FlowSessionKeys.flowSessionActive) else { return false }
if !FlowSessionPolicy.usesInactivityExpiry(defaults: store) {
return true
}
let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires) let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires)
return expires > Date().timeIntervalSince1970 return expires > Date().timeIntervalSince1970
} }
@@ -25,6 +25,18 @@ public enum FlowSessionPolicy {
inactivityDuration(defaults: defaults).timeInterval inactivityDuration(defaults: defaults).timeInterval
} }
public static func keepAliveMode(defaults: UserDefaults? = nil) -> FlowKeepAliveMode {
let store = resolvedDefaults(defaults)
return FlowKeepAliveMode.fromStored(
store.string(forKey: AppGroupConfiguration.Keys.flowKeepAliveMode)
)
}
/// PiP sessions have no inactivity expiry; only the Live Activity path times out.
public static func usesInactivityExpiry(defaults: UserDefaults? = nil) -> Bool {
keepAliveMode(defaults: defaults) == .liveActivity
}
private static func resolvedDefaults(_ defaults: UserDefaults?) -> UserDefaults { private static func resolvedDefaults(_ defaults: UserDefaults?) -> UserDefaults {
if let defaults { return defaults } if let defaults { return defaults }
guard let available = AppGroup.defaultsIfAvailable else { guard let available = AppGroup.defaultsIfAvailable else {
@@ -33,6 +33,7 @@ final class AppGroupConfigurationTests: XCTestCase {
XCTAssertEqual(config.polishIntensity, .default) XCTAssertEqual(config.polishIntensity, .default)
XCTAssertTrue(config.personalDictionary.entries.isEmpty) XCTAssertTrue(config.personalDictionary.entries.isEmpty)
XCTAssertTrue(config.flowSkipAppSwitch) XCTAssertTrue(config.flowSkipAppSwitch)
XCTAssertEqual(config.flowKeepAliveMode, .liveActivity)
XCTAssertEqual(config.flowInactivityDuration, .fiveMinutes) XCTAssertEqual(config.flowInactivityDuration, .fiveMinutes)
} }
@@ -58,6 +59,7 @@ final class AppGroupConfigurationTests: XCTestCase {
config.cursorDragNavigationEnabled = false config.cursorDragNavigationEnabled = false
config.polishIntensity = .light config.polishIntensity = .light
config.flowSkipAppSwitch = false config.flowSkipAppSwitch = false
config.flowKeepAliveMode = .pictureInPicture
// Use a non-default value so the round-trip actually proves persistence. // Use a non-default value so the round-trip actually proves persistence.
config.flowInactivityDuration = .threeHours config.flowInactivityDuration = .threeHours
config.save(to: defaults) config.save(to: defaults)
@@ -81,6 +83,7 @@ final class AppGroupConfigurationTests: XCTestCase {
XCTAssertFalse(loaded.cursorDragNavigationEnabled) XCTAssertFalse(loaded.cursorDragNavigationEnabled)
XCTAssertEqual(loaded.polishIntensity, .light) XCTAssertEqual(loaded.polishIntensity, .light)
XCTAssertFalse(loaded.flowSkipAppSwitch) XCTAssertFalse(loaded.flowSkipAppSwitch)
XCTAssertEqual(loaded.flowKeepAliveMode, .pictureInPicture)
XCTAssertEqual(loaded.flowInactivityDuration, .threeHours) XCTAssertEqual(loaded.flowInactivityDuration, .threeHours)
} }
@@ -29,6 +29,25 @@ final class FlowSessionPolicyTests: XCTestCase {
XCTAssertEqual(FlowInactivityDuration.tenMinutes.timeInterval, 10 * 60) XCTAssertEqual(FlowInactivityDuration.tenMinutes.timeInterval, 10 * 60)
} }
func testKeepAliveModeDefaultsToLiveActivity() {
let defaults = makeDefaults()
XCTAssertEqual(FlowSessionPolicy.keepAliveMode(defaults: defaults), .liveActivity)
XCTAssertTrue(FlowSessionPolicy.usesInactivityExpiry(defaults: defaults))
}
func testPiPSessionHasNoInactivityExpiry() {
let defaults = makeDefaults()
defaults.set(FlowKeepAliveMode.pictureInPicture.rawValue,
forKey: AppGroupConfiguration.Keys.flowKeepAliveMode)
FlowSessionBridge.markSessionActive(sessionId: UUID(), defaults: defaults)
XCTAssertTrue(FlowSessionBridge.isSessionActive(defaults: defaults))
XCTAssertNil(FlowSessionBridge.sessionExpiresAt(defaults: defaults))
FlowSessionBridge.touchLastActivity(defaults: defaults)
XCTAssertNil(FlowSessionBridge.sessionExpiresAt(defaults: defaults))
}
func testTouchLastActivityExtendsExpiry() { func testTouchLastActivityExtendsExpiry() {
let defaults = makeDefaults() let defaults = makeDefaults()
defaults.set(FlowInactivityDuration.tenMinutes.rawValue, forKey: AppGroupConfiguration.Keys.flowInactivityDuration) defaults.set(FlowInactivityDuration.tenMinutes.rawValue, forKey: AppGroupConfiguration.Keys.flowInactivityDuration)
@@ -62,6 +62,7 @@ final class SettingsCloudSyncTests: XCTestCase {
polishIntensity: SyncedField(value: .medium, updatedAt: stampA, deviceID: deviceA), polishIntensity: SyncedField(value: .medium, updatedAt: stampA, deviceID: deviceA),
llmThinkingEnabled: SyncedField(value: false, updatedAt: stampA, deviceID: deviceA), llmThinkingEnabled: SyncedField(value: false, updatedAt: stampA, deviceID: deviceA),
flowSkipAppSwitch: SyncedField(value: true, updatedAt: stampA, deviceID: deviceA), flowSkipAppSwitch: SyncedField(value: true, updatedAt: stampA, deviceID: deviceA),
flowKeepAliveMode: SyncedField(value: .liveActivity, updatedAt: stampA, deviceID: deviceA),
flowInactivityDuration: SyncedField(value: .twelveHours, updatedAt: stampA, deviceID: deviceA) flowInactivityDuration: SyncedField(value: .twelveHours, updatedAt: stampA, deviceID: deviceA)
) )
let remote = SyncedAppSettingsV2( let remote = SyncedAppSettingsV2(
@@ -82,6 +83,7 @@ final class SettingsCloudSyncTests: XCTestCase {
polishIntensity: SyncedField(value: .light, updatedAt: stampB, deviceID: deviceB), polishIntensity: SyncedField(value: .light, updatedAt: stampB, deviceID: deviceB),
llmThinkingEnabled: SyncedField(value: true, updatedAt: stampB, deviceID: deviceB), llmThinkingEnabled: SyncedField(value: true, updatedAt: stampB, deviceID: deviceB),
flowSkipAppSwitch: SyncedField(value: false, updatedAt: stampB, deviceID: deviceB), flowSkipAppSwitch: SyncedField(value: false, updatedAt: stampB, deviceID: deviceB),
flowKeepAliveMode: SyncedField(value: .pictureInPicture, updatedAt: stampB, deviceID: deviceB),
flowInactivityDuration: SyncedField(value: .threeHours, updatedAt: stampB, deviceID: deviceB) flowInactivityDuration: SyncedField(value: .threeHours, updatedAt: stampB, deviceID: deviceB)
) )
+1
View File
@@ -133,6 +133,7 @@ targets:
NSSpeechRecognitionUsageDescription: "OSGKeyboard uses speech recognition to transcribe your voice. Audio is processed on-device by default, or sent to your configured speech provider only when you enable cloud recognition." NSSpeechRecognitionUsageDescription: "OSGKeyboard uses speech recognition to transcribe your voice. Audio is processed on-device by default, or sent to your configured speech provider only when you enable cloud recognition."
UIBackgroundModes: UIBackgroundModes:
- audio - audio
- picture-in-picture
NSSupportsLiveActivities: true NSSupportsLiveActivities: true
NSAppTransportSecurity: NSAppTransportSecurity:
NSAllowsArbitraryLoads: false NSAllowsArbitraryLoads: false