Merge branch 'cursor/pip-keep-alive-2b89' into feature/polish-style-packs
Resolve conflicts in CHANGELOG (keep both entries) and FlowSessionManager (utterance PCM snapshot + PiP capture release after drain). Co-authored-by: Rocky <hkgood@users.noreply.github.com>
This commit is contained in:
@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Added
|
||||
- **Polish style packs**: choose a complete writing personality from the new iOS tab or Mac sidebar, create custom prompts, and sync selections and custom styles through iCloud. / **润色风格包**:可在 iOS 新 Tab 或 Mac 侧栏选择完整写作人格、创建自定义提示词,并通过 iCloud 同步选择与自定义风格。
|
||||
- **PiP Flow keep-alive**: Settings → Voice session lets you choose **Dynamic Island** (default, unchanged behaviour) or **Picture in Picture** — a live waveform PiP keeps the host alive with the mic released between utterances; closing PiP ends the session. / **PiP Flow 保活**:设置 → 语音会话可选 **灵动岛**(默认,行为不变)或 **画中画** — 实时波形 PiP 保活、句间释麦;关闭 PiP 即结束会话。
|
||||
- **Mac MLX streaming ASR**: local dictation uses Qwen3-ASR via mlx-audio-swift with overlay partial preview, tail drain, vocabulary prompt, and polish-before-insert. / **Mac MLX 流式 ASR**:本地听写改用 mlx-audio-swift 的 Qwen3-ASR,支持浮层 partial 预览、尾部截断、词库 prompt 与润色后再插入。
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
// FlowPictureInPictureController.swift
|
||||
// OSGKeyboard · Main App
|
||||
//
|
||||
// PiP keep-alive for Flow sessions: enqueues live waveform sample buffers
|
||||
// so the host process stays eligible for multitasking while the mic is off
|
||||
// between utterances.
|
||||
|
||||
import AVFoundation
|
||||
import AVKit
|
||||
import CoreMedia
|
||||
import UIKit
|
||||
|
||||
@MainActor
|
||||
final class FlowPictureInPictureController: NSObject {
|
||||
/// User closed the PiP window — host should end the Flow session.
|
||||
var onUserDismissed: (() -> Void)?
|
||||
|
||||
private(set) var isPictureInPictureActive = false
|
||||
|
||||
let displayLayer = AVSampleBufferDisplayLayer()
|
||||
|
||||
private var pipController: AVPictureInPictureController?
|
||||
private var displayLink: CADisplayLink?
|
||||
private weak var hostView: UIView?
|
||||
private var waveformLevels: [Float] = Array(repeating: 0, count: 24)
|
||||
private var isStoppingProgrammatically = false
|
||||
private var frameIndex: Int64 = 0
|
||||
|
||||
// MARK: - Host view
|
||||
|
||||
func attachHostView(_ view: UIView) {
|
||||
hostView = view
|
||||
displayLayer.frame = view.bounds
|
||||
displayLayer.videoGravity = .resizeAspectFill
|
||||
displayLayer.removeFromSuperlayer()
|
||||
view.layer.addSublayer(displayLayer)
|
||||
configureControllerIfNeeded()
|
||||
}
|
||||
|
||||
func updateHostLayoutIfNeeded() {
|
||||
guard let hostView else { return }
|
||||
displayLayer.frame = hostView.bounds
|
||||
}
|
||||
|
||||
// MARK: - Lifecycle
|
||||
|
||||
@discardableResult
|
||||
func start() -> Bool {
|
||||
guard AVPictureInPictureController.isPictureInPictureSupported() else {
|
||||
return false
|
||||
}
|
||||
configureControllerIfNeeded()
|
||||
startFramePump()
|
||||
guard pipController != nil else { return false }
|
||||
|
||||
if pipController?.isPictureInPictureActive == true {
|
||||
isPictureInPictureActive = true
|
||||
return true
|
||||
}
|
||||
|
||||
enqueueWaveformFrame()
|
||||
pipController?.startPictureInPicture()
|
||||
return true
|
||||
}
|
||||
|
||||
func startAndWait(timeout: TimeInterval = 4) async -> Bool {
|
||||
if isPictureInPictureActive { return true }
|
||||
guard start() else { return false }
|
||||
|
||||
let deadline = Date().addingTimeInterval(timeout)
|
||||
while Date() < deadline {
|
||||
if isPictureInPictureActive { return true }
|
||||
if pipController?.isPictureInPictureActive == true {
|
||||
isPictureInPictureActive = true
|
||||
return true
|
||||
}
|
||||
try? await Task.sleep(nanoseconds: 50_000_000)
|
||||
}
|
||||
return isPictureInPictureActive
|
||||
}
|
||||
|
||||
func stop() {
|
||||
isStoppingProgrammatically = true
|
||||
stopFramePump()
|
||||
pipController?.stopPictureInPicture()
|
||||
displayLayer.flushAndRemoveImage()
|
||||
isPictureInPictureActive = false
|
||||
isStoppingProgrammatically = false
|
||||
}
|
||||
|
||||
func updateWaveformLevels(_ levels: [Float]) {
|
||||
guard !levels.isEmpty else { return }
|
||||
waveformLevels = levels
|
||||
}
|
||||
|
||||
// MARK: - Private
|
||||
|
||||
private func configureControllerIfNeeded() {
|
||||
guard pipController == nil else { return }
|
||||
guard AVPictureInPictureController.isPictureInPictureSupported() else { return }
|
||||
|
||||
let contentSource = AVPictureInPictureController.ContentSource(
|
||||
sampleBufferDisplayLayer: displayLayer,
|
||||
playbackDelegate: self
|
||||
)
|
||||
let controller = AVPictureInPictureController(contentSource: contentSource)
|
||||
controller.delegate = self
|
||||
controller.canStartPictureInPictureAutomaticallyFromInline = true
|
||||
pipController = controller
|
||||
}
|
||||
|
||||
private func startFramePump() {
|
||||
guard displayLink == nil else { return }
|
||||
let link = CADisplayLink(target: self, selector: #selector(handleDisplayLink(_:)))
|
||||
link.preferredFrameRateRange = CAFrameRateRange(minimum: 20, maximum: 30, preferred: 24)
|
||||
link.add(to: .main, forMode: .common)
|
||||
displayLink = link
|
||||
}
|
||||
|
||||
private func stopFramePump() {
|
||||
displayLink?.invalidate()
|
||||
displayLink = nil
|
||||
}
|
||||
|
||||
@objc private func handleDisplayLink(_ link: CADisplayLink) {
|
||||
enqueueWaveformFrame()
|
||||
updateHostLayoutIfNeeded()
|
||||
|
||||
if let pipController, !pipController.isPictureInPictureActive, pipController.isPictureInPicturePossible {
|
||||
pipController.startPictureInPicture()
|
||||
}
|
||||
}
|
||||
|
||||
private func enqueueWaveformFrame() {
|
||||
guard let sampleBuffer = makeWaveformSampleBuffer(levels: resolvedLevels()) else { return }
|
||||
if displayLayer.status == .failed {
|
||||
displayLayer.flush()
|
||||
}
|
||||
displayLayer.enqueue(sampleBuffer)
|
||||
}
|
||||
|
||||
private func resolvedLevels() -> [Float] {
|
||||
if waveformLevels.contains(where: { $0 > 0.02 }) {
|
||||
return waveformLevels
|
||||
}
|
||||
// Idle breathing animation between utterances.
|
||||
let phase = Float(frameIndex) * 0.12
|
||||
return (0..<waveformLevels.count).map { index in
|
||||
let wave = sin(phase + Float(index) * 0.45)
|
||||
return max(0.04, 0.04 + wave * 0.03)
|
||||
}
|
||||
}
|
||||
|
||||
private func makeWaveformSampleBuffer(levels: [Float]) -> CMSampleBuffer? {
|
||||
let width = 320
|
||||
let height = 180
|
||||
frameIndex += 1
|
||||
|
||||
var pixelBuffer: CVPixelBuffer?
|
||||
let attrs: [String: Any] = [
|
||||
kCVPixelBufferCGImageCompatibilityKey as String: true,
|
||||
kCVPixelBufferCGBitmapContextCompatibilityKey as String: true,
|
||||
]
|
||||
let status = CVPixelBufferCreate(
|
||||
kCFAllocatorDefault,
|
||||
width,
|
||||
height,
|
||||
kCVPixelFormatType_32BGRA,
|
||||
attrs as CFDictionary,
|
||||
&pixelBuffer
|
||||
)
|
||||
guard status == kCVReturnSuccess, let pixelBuffer else { return nil }
|
||||
|
||||
CVPixelBufferLockBaseAddress(pixelBuffer, [])
|
||||
defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) }
|
||||
|
||||
guard let base = CVPixelBufferGetBaseAddress(pixelBuffer) else { return nil }
|
||||
let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
|
||||
let colorSpace = CGColorSpaceCreateDeviceRGB()
|
||||
guard let context = CGContext(
|
||||
data: base,
|
||||
width: width,
|
||||
height: height,
|
||||
bitsPerComponent: 8,
|
||||
bytesPerRow: bytesPerRow,
|
||||
space: colorSpace,
|
||||
bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue
|
||||
) else { return nil }
|
||||
|
||||
// Dark backdrop + accent waveform bars.
|
||||
context.setFillColor(UIColor(red: 0.07, green: 0.09, blue: 0.11, alpha: 1).cgColor)
|
||||
context.fill(CGRect(x: 0, y: 0, width: width, height: height))
|
||||
|
||||
let barCount = max(levels.count, 1)
|
||||
let barWidth = CGFloat(width) / CGFloat(barCount * 2)
|
||||
let accent = UIColor(red: 0.20, green: 0.78, blue: 0.55, alpha: 1)
|
||||
context.setFillColor(accent.cgColor)
|
||||
|
||||
for (index, level) in levels.enumerated() {
|
||||
let clamped = CGFloat(min(max(level, 0), 1))
|
||||
let barHeight = max(6, clamped * CGFloat(height) * 0.72)
|
||||
let x = (CGFloat(index) * 2 + 0.5) * barWidth
|
||||
let rect = CGRect(
|
||||
x: x,
|
||||
y: (CGFloat(height) - barHeight) / 2,
|
||||
width: barWidth,
|
||||
height: barHeight
|
||||
)
|
||||
let path = UIBezierPath(roundedRect: rect, cornerRadius: barWidth * 0.35)
|
||||
context.addPath(path.cgPath)
|
||||
context.fillPath()
|
||||
}
|
||||
|
||||
var formatDescription: CMFormatDescription?
|
||||
CMVideoFormatDescriptionCreateForImageBuffer(
|
||||
allocator: kCFAllocatorDefault,
|
||||
imageBuffer: pixelBuffer,
|
||||
formatDescriptionOut: &formatDescription
|
||||
)
|
||||
guard let formatDescription else { return nil }
|
||||
|
||||
var timing = CMSampleTimingInfo(
|
||||
duration: CMTime(value: 1, timescale: 24),
|
||||
presentationTimeStamp: CMTime(value: frameIndex, timescale: 24),
|
||||
decodeTimeStamp: .invalid
|
||||
)
|
||||
|
||||
var sampleBuffer: CMSampleBuffer?
|
||||
CMSampleBufferCreateForImageBuffer(
|
||||
allocator: kCFAllocatorDefault,
|
||||
imageBuffer: pixelBuffer,
|
||||
dataReady: true,
|
||||
makeDataReadyCallback: nil,
|
||||
refcon: nil,
|
||||
formatDescription: formatDescription,
|
||||
sampleTiming: &timing,
|
||||
sampleBufferOut: &sampleBuffer
|
||||
)
|
||||
return sampleBuffer
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - AVPictureInPictureControllerDelegate
|
||||
|
||||
extension FlowPictureInPictureController: AVPictureInPictureControllerDelegate {
|
||||
func pictureInPictureControllerDidStartPictureInPicture(
|
||||
_ pictureInPictureController: AVPictureInPictureController
|
||||
) {
|
||||
isPictureInPictureActive = true
|
||||
}
|
||||
|
||||
func pictureInPictureControllerDidStopPictureInPicture(
|
||||
_ pictureInPictureController: AVPictureInPictureController
|
||||
) {
|
||||
isPictureInPictureActive = false
|
||||
stopFramePump()
|
||||
guard !isStoppingProgrammatically else { return }
|
||||
onUserDismissed?()
|
||||
}
|
||||
|
||||
func pictureInPictureController(
|
||||
_ pictureInPictureController: AVPictureInPictureController,
|
||||
restoreUserInterfaceForPictureInPictureStopWithCompletionHandler completionHandler: @escaping (Bool) -> Void
|
||||
) {
|
||||
completionHandler(true)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - AVPictureInPictureSampleBufferPlaybackDelegate
|
||||
|
||||
extension FlowPictureInPictureController: AVPictureInPictureSampleBufferPlaybackDelegate {
|
||||
func pictureInPictureController(
|
||||
_ pictureInPictureController: AVPictureInPictureController,
|
||||
setPlaying playing: Bool
|
||||
) {
|
||||
if playing {
|
||||
startFramePump()
|
||||
} else {
|
||||
stopFramePump()
|
||||
}
|
||||
}
|
||||
|
||||
func pictureInPictureControllerTimeRangeForPlayback(
|
||||
_ pictureInPictureController: AVPictureInPictureController
|
||||
) -> CMTimeRange {
|
||||
CMTimeRange(start: .zero, duration: CMTime(value: 3600, timescale: 1))
|
||||
}
|
||||
|
||||
func pictureInPictureControllerIsPlaybackPaused(
|
||||
_ pictureInPictureController: AVPictureInPictureController
|
||||
) -> Bool {
|
||||
false
|
||||
}
|
||||
|
||||
func pictureInPictureController(
|
||||
_ pictureInPictureController: AVPictureInPictureController,
|
||||
didTransitionToRenderSize newRenderSize: CMVideoDimensions
|
||||
) {}
|
||||
|
||||
func pictureInPictureController(
|
||||
_ pictureInPictureController: AVPictureInPictureController,
|
||||
skipByInterval skipInterval: CMTime,
|
||||
completion completionHandler: @escaping () -> Void
|
||||
) {
|
||||
completionHandler()
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
@Published var coldStartContext: FlowColdStartContext?
|
||||
|
||||
private let capture = FlowContinuousCapture()
|
||||
private let pipController = FlowPictureInPictureController()
|
||||
private let store = AppGroupStore()
|
||||
/// Cloud-engine polish; local engine runs through built-in DeepSeek polish.
|
||||
private let polisher = PolishingService()
|
||||
@@ -82,6 +83,14 @@ final class FlowSessionManager: ObservableObject {
|
||||
private var coldStartRecoveryTask: Task<Void, Never>?
|
||||
/// Initial proof window — cold mic sessions often need >2.5s after app switch.
|
||||
private static let coldStartAudioProofTimeout: TimeInterval = 6
|
||||
|
||||
private var usesPiPKeepAlive: Bool {
|
||||
FlowSessionPolicy.keepAliveMode() == .pictureInPicture
|
||||
}
|
||||
|
||||
func attachPiPHostView(_ view: UIView) {
|
||||
pipController.attachHostView(view)
|
||||
}
|
||||
/// Guards the once-per-process launch reconciliation (scene reconnects
|
||||
/// recreate the `@StateObject`-owned manager within the same process).
|
||||
private static var didRunLaunchReconciliation = false
|
||||
@@ -126,6 +135,11 @@ final class FlowSessionManager: ObservableObject {
|
||||
kind: .recognitionInterrupted
|
||||
)
|
||||
}
|
||||
pipController.onUserDismissed = { [weak self] in
|
||||
guard let self, self.isActive else { return }
|
||||
self.debug("PiP dismissed by user — ending Flow session")
|
||||
self.endSession()
|
||||
}
|
||||
FlowTerminationCoordinator.register(self)
|
||||
}
|
||||
|
||||
@@ -280,7 +294,9 @@ final class FlowSessionManager: ObservableObject {
|
||||
// hit the same audio-proof timeout.
|
||||
coldStartRecoveryTask?.cancel()
|
||||
coldStartRecoveryTask = nil
|
||||
if capture.running {
|
||||
if usesPiPKeepAlive {
|
||||
pipController.stop()
|
||||
} else if capture.running {
|
||||
capture.stop()
|
||||
}
|
||||
sessionASR?.cancel()
|
||||
@@ -334,6 +350,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
if capture.running {
|
||||
capture.stop()
|
||||
}
|
||||
pipController.stop()
|
||||
|
||||
endBackgroundKeepAlive()
|
||||
ScreenWakeLock.release()
|
||||
@@ -406,6 +423,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
isUtteranceProcessing = false
|
||||
|
||||
capture.stop()
|
||||
pipController.stop()
|
||||
endBackgroundKeepAlive()
|
||||
ScreenWakeLock.release()
|
||||
sessionASR = nil
|
||||
@@ -422,6 +440,10 @@ final class FlowSessionManager: ObservableObject {
|
||||
}
|
||||
|
||||
func extendSession(duration: TimeInterval? = nil) {
|
||||
guard !usesPiPKeepAlive else {
|
||||
refreshHostReady()
|
||||
return
|
||||
}
|
||||
let resolved = duration ?? FlowSessionPolicy.sessionDuration()
|
||||
FlowSessionBridge.extendSession(by: resolved)
|
||||
sessionExpiresAt = Date().addingTimeInterval(resolved)
|
||||
@@ -495,6 +517,10 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
private func reactivateCaptureIfNeeded() async {
|
||||
guard isActive else { return }
|
||||
if usesPiPKeepAlive, !isUtteranceRecording, !isUtteranceProcessing, !capture.running {
|
||||
refreshHostReady()
|
||||
return
|
||||
}
|
||||
// A system interruption (call / Siri) may be in progress. Probe it:
|
||||
// `setActive(true)` inside `reassertIfRunning` fails while the
|
||||
// interruption is live and succeeds once it ends — which also covers
|
||||
@@ -565,12 +591,22 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
let pollingAlive = pollingTask != nil && pollingTask?.isCancelled != true
|
||||
let hasRecentAudio = capture.engineHasRecentAudio(maxAge: 2)
|
||||
let canAcceptUtterance = capture.engineIsLive
|
||||
let canAcceptUtterance: Bool
|
||||
if usesPiPKeepAlive {
|
||||
canAcceptUtterance = pipController.isPictureInPictureActive
|
||||
&& pollingAlive
|
||||
&& !isUtteranceRecording
|
||||
&& !isUtteranceProcessing
|
||||
&& sessionWarning == nil
|
||||
&& !capture.isInterrupted
|
||||
} else {
|
||||
canAcceptUtterance = capture.engineIsLive
|
||||
&& pollingAlive
|
||||
&& hasRecentAudio
|
||||
&& !isUtteranceRecording
|
||||
&& !isUtteranceProcessing
|
||||
&& sessionWarning == nil
|
||||
}
|
||||
|
||||
let reason: FlowReadySnapshot.Reason
|
||||
if canAcceptUtterance {
|
||||
@@ -581,9 +617,11 @@ final class FlowSessionManager: ObservableObject {
|
||||
reason = .recording
|
||||
} else if isUtteranceProcessing {
|
||||
reason = .processing
|
||||
} else if !capture.engineIsLive {
|
||||
} else if usesPiPKeepAlive, !pipController.isPictureInPictureActive {
|
||||
reason = .starting
|
||||
} else if !usesPiPKeepAlive, !capture.engineIsLive {
|
||||
reason = .audioEngineNotLive
|
||||
} else if !hasRecentAudio {
|
||||
} else if !usesPiPKeepAlive, !hasRecentAudio {
|
||||
reason = .waitingForAudioProof
|
||||
} else {
|
||||
reason = .starting
|
||||
@@ -656,6 +694,11 @@ final class FlowSessionManager: ObservableObject {
|
||||
/// custom keyboard extension sees green immediately.
|
||||
func refreshForInlineKeyboardFocus() async {
|
||||
guard isActive else { return }
|
||||
if usesPiPKeepAlive {
|
||||
refreshHostReady()
|
||||
FlowSessionBridge.writeHeartbeat()
|
||||
return
|
||||
}
|
||||
await reactivateCaptureIfNeeded()
|
||||
refreshHostReady()
|
||||
if !FlowSessionBridge.isHostReady() {
|
||||
@@ -668,7 +711,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
/// Extend expiry after utterance completion based on the inactivity policy.
|
||||
private func touchSessionActivity() {
|
||||
guard isActive else { return }
|
||||
guard isActive, !usesPiPKeepAlive else { return }
|
||||
FlowSessionBridge.touchLastActivity()
|
||||
if let expires = FlowSessionBridge.sessionExpiresAt() {
|
||||
sessionExpiresAt = Date(timeIntervalSince1970: expires)
|
||||
@@ -699,6 +742,25 @@ final class FlowSessionManager: ObservableObject {
|
||||
return
|
||||
}
|
||||
|
||||
if usesPiPKeepAlive {
|
||||
let pipReady = await pipController.startAndWait()
|
||||
guard pipReady else {
|
||||
let message = AppL10n.string("flow.pip.error.unavailable")
|
||||
sessionWarning = message
|
||||
traceState("startSessionAsync.failed", extra: "reason=pipUnavailable")
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
if isColdStartHandoff {
|
||||
showColdStartAudioFailure(message: message)
|
||||
}
|
||||
debug("PiP keep-alive failed to start")
|
||||
return
|
||||
}
|
||||
activateFlowSessionAfterPiPProof(duration: duration)
|
||||
traceState("startSessionAsync.ready")
|
||||
debug("Flow session started (PiP keep-alive), mic released between utterances")
|
||||
return
|
||||
}
|
||||
|
||||
do {
|
||||
try capture.start()
|
||||
} catch {
|
||||
@@ -735,6 +797,31 @@ final class FlowSessionManager: ObservableObject {
|
||||
debug("Flow session started (\(Int(duration ?? FlowSessionPolicy.sessionDuration()))s inactivity window), continuous capture running")
|
||||
}
|
||||
|
||||
private func activateFlowSessionAfterPiPProof(duration: TimeInterval?) {
|
||||
let sessionId = activeSessionId ?? UUID()
|
||||
activeSessionId = sessionId
|
||||
lastHandledCommandSeq = 0
|
||||
FlowSessionBridge.markSessionActive(duration: duration, sessionId: sessionId)
|
||||
FlowSessionDarwin.postSessionChanged()
|
||||
isActive = true
|
||||
ScreenWakeLock.acquire()
|
||||
sessionExpiresAt = nil
|
||||
|
||||
startHeartbeat()
|
||||
startCommandObserver()
|
||||
startPolling()
|
||||
startLevelPublishing()
|
||||
expiryTask?.cancel()
|
||||
expiryTask = nil
|
||||
|
||||
bindSessionASRIfNeeded()
|
||||
scheduleASRWarmup()
|
||||
FlowLiveActivityController.startSession()
|
||||
|
||||
refreshHostReady()
|
||||
traceState("activateFlowSessionAfterPiPProof.done")
|
||||
}
|
||||
|
||||
private func activateFlowSessionAfterAudioProof(duration: TimeInterval?) {
|
||||
let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration()
|
||||
let sessionId = activeSessionId ?? UUID()
|
||||
@@ -762,6 +849,12 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
private func prepareExistingSessionForColdStartReturn() async {
|
||||
guard isColdStartHandoff, isActive else { return }
|
||||
if usesPiPKeepAlive {
|
||||
sessionWarning = nil
|
||||
refreshHostReady()
|
||||
handleColdStartAfterSessionReady()
|
||||
return
|
||||
}
|
||||
await reactivateCaptureIfNeeded()
|
||||
guard await waitForAudioProof() else {
|
||||
let message = AppL10n.string("flow.coldStart.error.audioTimeout")
|
||||
@@ -815,7 +908,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
}
|
||||
|
||||
private func scheduleAutoReturnToHostIfNeeded(hostEntry: HostAppEntry?) {
|
||||
let skipSwitch = FlowSessionPolicy.skipAppSwitch()
|
||||
let skipSwitch = usesPiPKeepAlive || FlowSessionPolicy.skipAppSwitch()
|
||||
guard skipSwitch, hostEntry != nil else { return }
|
||||
Task { @MainActor [weak self] in
|
||||
try? await Task.sleep(nanoseconds: 450_000_000)
|
||||
@@ -835,6 +928,21 @@ final class FlowSessionManager: ObservableObject {
|
||||
coldStartRecoveryTask?.cancel()
|
||||
coldStartRecoveryTask = Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
if self.usesPiPKeepAlive {
|
||||
let recovered = await self.pipController.startAndWait()
|
||||
self.traceState("coldStartRecovery.pip", extra: "recovered=\(recovered)")
|
||||
guard !Task.isCancelled, self.isColdStartHandoff else { return }
|
||||
if recovered {
|
||||
if self.isActive {
|
||||
self.refreshHostReady()
|
||||
self.handleColdStartAfterSessionReady()
|
||||
} else {
|
||||
self.activateFlowSessionAfterPiPProof(duration: duration)
|
||||
self.handleColdStartAfterSessionReady()
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
var recovered = false
|
||||
for attempt in 1...3 {
|
||||
guard !Task.isCancelled, self.isColdStartHandoff else { return }
|
||||
@@ -1006,7 +1114,12 @@ final class FlowSessionManager: ObservableObject {
|
||||
switch command.action {
|
||||
case .startRecording:
|
||||
guard !isUtteranceRecording, !isUtteranceProcessing else { return }
|
||||
beginUtterance(utteranceId: command.utteranceId, commandSeq: command.commandSeq)
|
||||
Task { @MainActor [weak self] in
|
||||
await self?.handleStartRecordingCommand(
|
||||
utteranceId: command.utteranceId,
|
||||
commandSeq: command.commandSeq
|
||||
)
|
||||
}
|
||||
case .stopRecording:
|
||||
guard currentUtteranceId == command.utteranceId else { return }
|
||||
if isUtteranceRecording {
|
||||
@@ -1076,6 +1189,42 @@ final class FlowSessionManager: ObservableObject {
|
||||
)
|
||||
}
|
||||
|
||||
private func handleStartRecordingCommand(utteranceId: UUID?, commandSeq: Int64) async {
|
||||
if usesPiPKeepAlive {
|
||||
refreshHostReady()
|
||||
let micReady = await ensureCaptureReadyForPiPUtterance()
|
||||
guard micReady else {
|
||||
failUtterance(
|
||||
message: AppL10n.string("flow.coldStart.error.audioTimeout"),
|
||||
kind: .audioUnavailable
|
||||
)
|
||||
return
|
||||
}
|
||||
}
|
||||
beginUtterance(utteranceId: utteranceId, commandSeq: commandSeq)
|
||||
}
|
||||
|
||||
private func ensureCaptureReadyForPiPUtterance() async -> Bool {
|
||||
if capture.engineHasRecentAudio(maxAge: 2) {
|
||||
return true
|
||||
}
|
||||
do {
|
||||
try capture.start()
|
||||
} catch {
|
||||
debug("PiP utterance capture start failed: \(error.localizedDescription)")
|
||||
return false
|
||||
}
|
||||
return await capture.awaitAudioFlowing(timeout: Self.coldStartAudioProofTimeout)
|
||||
}
|
||||
|
||||
private func releaseCaptureAfterPiPUtteranceIfNeeded() {
|
||||
guard usesPiPKeepAlive, capture.running else { return }
|
||||
guard !isUtteranceRecording, !isUtteranceProcessing else { return }
|
||||
capture.stop()
|
||||
pipController.updateWaveformLevels([])
|
||||
refreshHostReady()
|
||||
}
|
||||
|
||||
private func beginUtterance(utteranceId: UUID? = nil, commandSeq: Int64 = 0) {
|
||||
guard capture.engineHasRecentAudio(maxAge: 2) else {
|
||||
traceState("beginUtterance.blocked", extra: "reason=audioNotRecent")
|
||||
@@ -1219,6 +1368,10 @@ final class FlowSessionManager: ObservableObject {
|
||||
let drainReport = await self.capture.endUtteranceAndDrain()
|
||||
FlowDiagnostics.logDrain(drainReport)
|
||||
self.utterancePCMSamples = self.capture.consumeUtteranceSamples()
|
||||
if self.usesPiPKeepAlive {
|
||||
self.capture.stop()
|
||||
self.pipController.updateWaveformLevels([])
|
||||
}
|
||||
await self.finalizeUtterance(
|
||||
sessionId: drainingSessionId,
|
||||
utteranceId: drainingUtteranceId,
|
||||
@@ -1241,6 +1394,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
chunkedPipeline = nil
|
||||
asr.cancel()
|
||||
capture.cancelUtterance()
|
||||
releaseCaptureAfterPiPUtteranceIfNeeded()
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
bestPartialSnapshot = ""
|
||||
@@ -1269,6 +1423,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
chunkedPipeline = nil
|
||||
asr.cancel()
|
||||
capture.cancelUtterance()
|
||||
releaseCaptureAfterPiPUtteranceIfNeeded()
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
bestPartialSnapshot = ""
|
||||
@@ -1293,6 +1448,8 @@ final class FlowSessionManager: ObservableObject {
|
||||
finalizeTask?.cancel()
|
||||
finalizeTask = nil
|
||||
chunkedPipeline = nil
|
||||
capture.cancelUtterance()
|
||||
releaseCaptureAfterPiPUtteranceIfNeeded()
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
bestPartialSnapshot = ""
|
||||
@@ -1663,6 +1820,9 @@ final class FlowSessionManager: ObservableObject {
|
||||
while !Task.isCancelled {
|
||||
guard let self, self.isActive else { break }
|
||||
let levels = self.capture.currentAudioLevels()
|
||||
if self.usesPiPKeepAlive {
|
||||
self.pipController.updateWaveformLevels(levels)
|
||||
}
|
||||
if levels.contains(where: { $0 > 0 }) {
|
||||
FlowSessionBridge.storeAudioLevels(levels)
|
||||
}
|
||||
@@ -1686,8 +1846,13 @@ final class FlowSessionManager: ObservableObject {
|
||||
while !Task.isCancelled {
|
||||
guard let self else { break }
|
||||
if self.isActive, !self.capture.engineIsLive {
|
||||
let shouldReassert = !self.usesPiPKeepAlive
|
||||
|| self.isUtteranceRecording
|
||||
|| self.isUtteranceProcessing
|
||||
if shouldReassert {
|
||||
await self.reactivateCaptureIfNeeded()
|
||||
}
|
||||
}
|
||||
FlowSessionBridge.writeHeartbeat()
|
||||
self.refreshHostReady()
|
||||
tick += 1
|
||||
@@ -1701,6 +1866,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
}
|
||||
|
||||
private func scheduleExpiry(after duration: TimeInterval) {
|
||||
guard !usesPiPKeepAlive else { return }
|
||||
expiryTask?.cancel()
|
||||
expiryTask = Task { @MainActor [weak self] in
|
||||
try? await Task.sleep(nanoseconds: UInt64(duration * 1_000_000_000))
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
// FlowPiPHostView.swift
|
||||
// OSGKeyboard · Main App
|
||||
//
|
||||
// Hidden host for the PiP sample-buffer display layer (must live in the window hierarchy).
|
||||
|
||||
import SwiftUI
|
||||
import UIKit
|
||||
|
||||
struct FlowPiPHostView: UIViewRepresentable {
|
||||
let attach: (UIView) -> Void
|
||||
|
||||
func makeUIView(context: Context) -> UIView {
|
||||
let view = UIView(frame: CGRect(x: 0, y: 0, width: 2, height: 2))
|
||||
view.isUserInteractionEnabled = false
|
||||
view.backgroundColor = .clear
|
||||
attach(view)
|
||||
return view
|
||||
}
|
||||
|
||||
func updateUIView(_ uiView: UIView, context: Context) {
|
||||
attach(uiView)
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,14 @@ struct MainAppRoot: View {
|
||||
}
|
||||
}
|
||||
.animation(.easeInOut(duration: 0.2), value: flowManager.coldStartContext != nil)
|
||||
.background {
|
||||
FlowPiPHostView { view in
|
||||
flowManager.attachPiPHostView(view)
|
||||
}
|
||||
.frame(width: 2, height: 2)
|
||||
.opacity(0.001)
|
||||
.allowsHitTesting(false)
|
||||
}
|
||||
.onAppear {
|
||||
flowManager.setAppForeground(scenePhase == .active)
|
||||
// Register the URL handler BEFORE the foreground auto-start.
|
||||
|
||||
@@ -29,6 +29,8 @@ struct SettingsView: View {
|
||||
// Dynamic locale list loaded from SFSpeechRecognizer on first appear.
|
||||
@State private var dynamicLocales: [(id: String, onDevice: Bool)] = []
|
||||
@State private var showResetConfirmation = false
|
||||
@State private var showActiveFlowSessionAlert = false
|
||||
@State private var pendingKeepAliveMode: FlowKeepAliveMode?
|
||||
// v0.2.0: no on-device model manager / pending download state —
|
||||
// iOS `SpeechAnalyzer` ships with iOS 26 and needs nothing
|
||||
// downloaded.
|
||||
@@ -105,7 +107,58 @@ struct SettingsView: View {
|
||||
VStack(alignment: .leading, spacing: SettingsListMetrics.sectionLabelSpacing) {
|
||||
sectionHeader("settings.flow.title")
|
||||
VStack(spacing: 0) {
|
||||
FlowKeepAliveModePickerRow(
|
||||
selection: Binding(
|
||||
get: { config.flowKeepAliveMode },
|
||||
set: { newMode in
|
||||
applyKeepAliveModeChange(newMode)
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
if config.flowKeepAliveMode == .liveActivity {
|
||||
Divider().background(palette.divider)
|
||||
|
||||
FlowInactivityPickerRow(
|
||||
selection: Binding(
|
||||
get: { config.flowInactivityDuration },
|
||||
set: { config.flowInactivityDuration = $0 }
|
||||
)
|
||||
)
|
||||
|
||||
Divider().background(palette.divider)
|
||||
|
||||
Toggle(isOn: $config.flowSkipAppSwitch) {
|
||||
flowSkipAppSwitchLabel
|
||||
}
|
||||
.tint(palette.accent)
|
||||
.settingsListRow()
|
||||
} else {
|
||||
Divider().background(palette.divider)
|
||||
|
||||
Text("settings.flow.keepAlive.pictureInPicture.note")
|
||||
.font(TypeStyle.caption2)
|
||||
.foregroundStyle(palette.textTertiary)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.settingsListRow()
|
||||
}
|
||||
}
|
||||
.background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.xl, style: .continuous))
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: Radius.xl, style: .continuous)
|
||||
.stroke(palette.divider, lineWidth: 0.5)
|
||||
)
|
||||
}
|
||||
.alert("settings.flow.keepAlive.activeSession.title", isPresented: $showActiveFlowSessionAlert) {
|
||||
Button("common.done", role: .cancel) {
|
||||
pendingKeepAliveMode = nil
|
||||
}
|
||||
} message: {
|
||||
Text("settings.flow.keepAlive.activeSession.message")
|
||||
}
|
||||
}
|
||||
|
||||
private var flowSkipAppSwitchLabel: some View {
|
||||
VStack(alignment: .leading, spacing: Spacing.xxs) {
|
||||
Text("settings.flow.skipAppSwitch.title")
|
||||
.font(TypeStyle.body)
|
||||
@@ -115,24 +168,15 @@ struct SettingsView: View {
|
||||
.foregroundStyle(palette.textTertiary)
|
||||
}
|
||||
}
|
||||
.tint(palette.accent)
|
||||
.settingsListRow()
|
||||
|
||||
Divider().background(palette.divider)
|
||||
|
||||
FlowInactivityPickerRow(
|
||||
selection: Binding(
|
||||
get: { config.flowInactivityDuration },
|
||||
set: { config.flowInactivityDuration = $0 }
|
||||
)
|
||||
)
|
||||
}
|
||||
.background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.xl, style: .continuous))
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: Radius.xl, style: .continuous)
|
||||
.stroke(palette.divider, lineWidth: 0.5)
|
||||
)
|
||||
private func applyKeepAliveModeChange(_ newMode: FlowKeepAliveMode) {
|
||||
guard newMode != config.flowKeepAliveMode else { return }
|
||||
if FlowSessionBridge.isSessionActive() {
|
||||
pendingKeepAliveMode = newMode
|
||||
showActiveFlowSessionAlert = true
|
||||
return
|
||||
}
|
||||
config.flowKeepAliveMode = newMode
|
||||
}
|
||||
|
||||
// MARK: - Engine
|
||||
@@ -520,6 +564,31 @@ private struct AppearancePickerRow: View {
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Flow keep-alive mode picker row
|
||||
|
||||
private struct FlowKeepAliveModePickerRow: View {
|
||||
@Binding var selection: FlowKeepAliveMode
|
||||
|
||||
private var options: [(id: String, label: String)] {
|
||||
FlowKeepAliveMode.allCases.map { mode in
|
||||
(mode.rawValue, AppL10n.string(mode.labelKey))
|
||||
}
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
PickerRow(
|
||||
title: AppL10n.string("settings.flow.keepAlive.title"),
|
||||
options: options,
|
||||
selection: Binding(
|
||||
get: { selection.rawValue },
|
||||
set: { newValue in
|
||||
selection = FlowKeepAliveMode(rawValue: newValue) ?? .liveActivity
|
||||
}
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Flow inactivity picker row
|
||||
|
||||
private struct FlowInactivityPickerRow: View {
|
||||
|
||||
@@ -449,6 +449,14 @@
|
||||
|
||||
/* Flow session policy */
|
||||
"settings.flow.title" = "Voice session";
|
||||
"settings.flow.keepAlive.title" = "Keep-alive mode";
|
||||
"settings.flow.keepAlive.liveActivity" = "Dynamic Island";
|
||||
"settings.flow.keepAlive.liveActivity.subtitle" = "Continuous mic session with inactivity timeout.";
|
||||
"settings.flow.keepAlive.pictureInPicture" = "Picture in Picture";
|
||||
"settings.flow.keepAlive.pictureInPicture.subtitle" = "Waveform PiP keeps the app alive; mic is released between utterances.";
|
||||
"settings.flow.keepAlive.pictureInPicture.note" = "Picture in Picture stays active until you close it. Skip app switch is always on in this mode.";
|
||||
"settings.flow.keepAlive.activeSession.title" = "End the current session first";
|
||||
"settings.flow.keepAlive.activeSession.message" = "Stop the active voice session before changing keep-alive mode.";
|
||||
"settings.flow.skipAppSwitch.title" = "Skip app switch";
|
||||
"settings.flow.skipAppSwitch.subtitle" = "After a cold start, try to return to the app you came from.";
|
||||
"settings.flow.inactivity.title" = "End session after inactivity";
|
||||
@@ -469,6 +477,7 @@
|
||||
"flow.coldStart.permission.title" = "Permission required";
|
||||
"flow.coldStart.audio.title" = "Voice could not start";
|
||||
"flow.coldStart.error.audioTimeout" = "The microphone did not become ready in time. It may be busy in another app. Please try again.";
|
||||
"flow.pip.error.unavailable" = "Picture in Picture could not start. Check that PiP is allowed for OSGKeyboard in Settings.";
|
||||
"flow.coldStart.action.settings" = "Open Settings";
|
||||
"flow.coldStart.action.retry" = "Try Again";
|
||||
"flow.coldStart.swipeHint" = "Swipe right along the bottom bar\nto return to your previous app.";
|
||||
|
||||
@@ -448,6 +448,14 @@
|
||||
|
||||
/* Flow 会话策略 */
|
||||
"settings.flow.title" = "语音会话";
|
||||
"settings.flow.keepAlive.title" = "保活方式";
|
||||
"settings.flow.keepAlive.liveActivity" = "灵动岛";
|
||||
"settings.flow.keepAlive.liveActivity.subtitle" = "麦克风常驻,可按无活动时长结束会话。";
|
||||
"settings.flow.keepAlive.pictureInPicture" = "画中画";
|
||||
"settings.flow.keepAlive.pictureInPicture.subtitle" = "波形画中画保活;句间释放麦克风。";
|
||||
"settings.flow.keepAlive.pictureInPicture.note" = "画中画将持续保活,直到你关闭小窗。此模式下始终跳过应用切换。";
|
||||
"settings.flow.keepAlive.activeSession.title" = "请先结束当前会话";
|
||||
"settings.flow.keepAlive.activeSession.message" = "更改保活方式前,请先结束正在进行的语音会话。";
|
||||
"settings.flow.skipAppSwitch.title" = "跳过应用切换";
|
||||
"settings.flow.skipAppSwitch.subtitle" = "冷启动完成后,尝试自动返回来源 App。";
|
||||
"settings.flow.inactivity.title" = "无活动后结束会话";
|
||||
@@ -468,6 +476,7 @@
|
||||
"flow.coldStart.permission.title" = "需要权限";
|
||||
"flow.coldStart.audio.title" = "语音暂时无法启动";
|
||||
"flow.coldStart.error.audioTimeout" = "麦克风未能及时就绪,可能正被通话或其他 App 占用。请稍后重试。";
|
||||
"flow.pip.error.unavailable" = "无法启动画中画,请在系统设置中允许 OSGKeyboard 使用画中画。";
|
||||
"flow.coldStart.action.settings" = "前往设置";
|
||||
"flow.coldStart.action.retry" = "重试";
|
||||
"flow.coldStart.swipeHint" = "沿屏幕底部的横条,从左向右滑动\n返回上一个 App。";
|
||||
|
||||
@@ -56,6 +56,8 @@ public struct AppGroupConfiguration: Sendable, Equatable {
|
||||
public static let settingsCloudPayloadV2 = "config.settings.cloudPayload.v2"
|
||||
/// When true, the host app auto-returns to the source app after a cold-start handoff.
|
||||
public static let flowSkipAppSwitch = "config.flowSkipAppSwitch"
|
||||
/// Raw `FlowKeepAliveMode` value; mutually exclusive PiP vs Live Activity path.
|
||||
public static let flowKeepAliveMode = "config.flowKeepAliveMode"
|
||||
/// Raw `FlowInactivityDuration` value; session expires after this idle window.
|
||||
public static let flowInactivityDuration = "config.flowInactivityDuration"
|
||||
/// One-shot: remap previous product defaults (30m / 10m) → 5m.
|
||||
@@ -96,6 +98,8 @@ public struct AppGroupConfiguration: Sendable, Equatable {
|
||||
public var settingsICloudSyncEnabled: Bool
|
||||
/// Auto-return to the host app after `startflow` cold start (default on).
|
||||
public var flowSkipAppSwitch: Bool
|
||||
/// PiP vs Live Activity keep-alive strategy (mutually exclusive).
|
||||
public var flowKeepAliveMode: FlowKeepAliveMode
|
||||
/// Idle timeout before the Flow session ends; resets on each utterance.
|
||||
public var flowInactivityDuration: FlowInactivityDuration
|
||||
/// Whether local `SpeechAnalyzer` should attach the prepared custom language model.
|
||||
@@ -273,6 +277,9 @@ public struct AppGroupConfiguration: Sendable, Equatable {
|
||||
}
|
||||
return defaults.bool(forKey: Keys.flowSkipAppSwitch)
|
||||
}(),
|
||||
flowKeepAliveMode: FlowKeepAliveMode.fromStored(
|
||||
defaults.string(forKey: Keys.flowKeepAliveMode)
|
||||
),
|
||||
flowInactivityDuration: FlowInactivityDuration.fromStored(
|
||||
defaults.string(forKey: Keys.flowInactivityDuration)
|
||||
),
|
||||
@@ -383,6 +390,7 @@ public struct AppGroupConfiguration: Sendable, Equatable {
|
||||
defaults.set(llmThinkingEnabled, forKey: Keys.llmThinkingEnabled)
|
||||
defaults.set(activePolishStyleId, forKey: Keys.activePolishStyleId)
|
||||
defaults.set(flowSkipAppSwitch, forKey: Keys.flowSkipAppSwitch)
|
||||
defaults.set(flowKeepAliveMode.rawValue, forKey: Keys.flowKeepAliveMode)
|
||||
defaults.set(flowInactivityDuration.rawValue, forKey: Keys.flowInactivityDuration)
|
||||
defaults.set(localASRCustomLanguageModelEnabled, forKey: Keys.localASRCustomLanguageModelEnabled)
|
||||
defaults.set(personalDictionaryICloudSyncEnabled, forKey: Keys.personalDictionaryICloudSyncEnabled)
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
// FlowKeepAliveMode.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// User-selectable Flow session keep-alive strategy (mutually exclusive).
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum FlowKeepAliveMode: String, CaseIterable, Identifiable, Sendable, Codable {
|
||||
/// Continuous audio capture + Live Activity (current default behaviour).
|
||||
case liveActivity = "liveActivity"
|
||||
/// Picture-in-picture waveform keep-alive; mic released between utterances.
|
||||
case pictureInPicture = "pictureInPicture"
|
||||
|
||||
public var id: String { rawValue }
|
||||
|
||||
/// Existing installs keep the Live Activity / continuous-capture path.
|
||||
public static let `default`: FlowKeepAliveMode = .liveActivity
|
||||
|
||||
public var labelKey: String {
|
||||
switch self {
|
||||
case .liveActivity: return "settings.flow.keepAlive.liveActivity"
|
||||
case .pictureInPicture: return "settings.flow.keepAlive.pictureInPicture"
|
||||
}
|
||||
}
|
||||
|
||||
public var subtitleKey: String {
|
||||
switch self {
|
||||
case .liveActivity: return "settings.flow.keepAlive.liveActivity.subtitle"
|
||||
case .pictureInPicture: return "settings.flow.keepAlive.pictureInPicture.subtitle"
|
||||
}
|
||||
}
|
||||
|
||||
public static func fromStored(_ raw: String?) -> FlowKeepAliveMode {
|
||||
guard let raw, let value = FlowKeepAliveMode(rawValue: raw) else {
|
||||
return .default
|
||||
}
|
||||
return value
|
||||
}
|
||||
}
|
||||
@@ -237,7 +237,22 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
/// Idle window before an active Flow session expires; resets on each utterance.
|
||||
/// PiP vs Live Activity keep-alive (mutually exclusive).
|
||||
@Published public var flowKeepAliveMode: FlowKeepAliveMode {
|
||||
didSet {
|
||||
guard !isApplyingConfiguration, flowKeepAliveMode != configuration.flowKeepAliveMode else { return }
|
||||
configuration.flowKeepAliveMode = flowKeepAliveMode
|
||||
if flowKeepAliveMode == .pictureInPicture {
|
||||
configuration.flowSkipAppSwitch = true
|
||||
if flowSkipAppSwitch != true {
|
||||
flowSkipAppSwitch = true
|
||||
}
|
||||
}
|
||||
persistConfiguration()
|
||||
}
|
||||
}
|
||||
|
||||
/// Idle window before an active Flow session expires; Live Activity mode only.
|
||||
@Published public var flowInactivityDuration: FlowInactivityDuration {
|
||||
didSet {
|
||||
guard !isApplyingConfiguration,
|
||||
@@ -347,6 +362,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
|
||||
polishIntensity = configuration.polishIntensity
|
||||
llmThinkingEnabled = configuration.llmThinkingEnabled
|
||||
flowSkipAppSwitch = configuration.flowSkipAppSwitch
|
||||
flowKeepAliveMode = configuration.flowKeepAliveMode
|
||||
flowInactivityDuration = configuration.flowInactivityDuration
|
||||
localASRCustomLanguageModelEnabled = configuration.localASRCustomLanguageModelEnabled
|
||||
isSyncingProviderAPIKey = true
|
||||
@@ -433,6 +449,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
|
||||
polishIntensity = fresh.polishIntensity
|
||||
llmThinkingEnabled = fresh.llmThinkingEnabled
|
||||
flowSkipAppSwitch = fresh.flowSkipAppSwitch
|
||||
flowKeepAliveMode = fresh.flowKeepAliveMode
|
||||
flowInactivityDuration = fresh.flowInactivityDuration
|
||||
localASRCustomLanguageModelEnabled = fresh.localASRCustomLanguageModelEnabled
|
||||
isSyncingProviderAPIKey = true
|
||||
|
||||
@@ -30,6 +30,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
|
||||
public var activePolishStyleId: SyncedField<String>
|
||||
public var llmThinkingEnabled: SyncedField<Bool>
|
||||
public var flowSkipAppSwitch: SyncedField<Bool>
|
||||
public var flowKeepAliveMode: SyncedField<FlowKeepAliveMode>
|
||||
public var flowInactivityDuration: SyncedField<FlowInactivityDuration>
|
||||
|
||||
public init(
|
||||
@@ -52,6 +53,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
|
||||
activePolishStyleId: SyncedField<String>,
|
||||
llmThinkingEnabled: SyncedField<Bool>,
|
||||
flowSkipAppSwitch: SyncedField<Bool>,
|
||||
flowKeepAliveMode: SyncedField<FlowKeepAliveMode>,
|
||||
flowInactivityDuration: SyncedField<FlowInactivityDuration>
|
||||
) {
|
||||
self.schemaVersion = schemaVersion
|
||||
@@ -73,6 +75,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
|
||||
self.activePolishStyleId = activePolishStyleId
|
||||
self.llmThinkingEnabled = llmThinkingEnabled
|
||||
self.flowSkipAppSwitch = flowSkipAppSwitch
|
||||
self.flowKeepAliveMode = flowKeepAliveMode
|
||||
self.flowInactivityDuration = flowInactivityDuration
|
||||
}
|
||||
|
||||
@@ -96,6 +99,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
|
||||
case activePolishStyleId
|
||||
case llmThinkingEnabled
|
||||
case flowSkipAppSwitch
|
||||
case flowKeepAliveMode
|
||||
case flowInactivityDuration
|
||||
}
|
||||
|
||||
@@ -136,6 +140,14 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
|
||||
forKey: .llmThinkingEnabled
|
||||
) ?? SyncedField(value: false, updatedAt: polishIntensity.updatedAt, deviceID: polishIntensity.deviceID)
|
||||
flowSkipAppSwitch = try container.decode(SyncedField<Bool>.self, forKey: .flowSkipAppSwitch)
|
||||
flowKeepAliveMode = try container.decodeIfPresent(
|
||||
SyncedField<FlowKeepAliveMode>.self,
|
||||
forKey: .flowKeepAliveMode
|
||||
) ?? SyncedField(
|
||||
value: .liveActivity,
|
||||
updatedAt: flowSkipAppSwitch.updatedAt,
|
||||
deviceID: flowSkipAppSwitch.deviceID
|
||||
)
|
||||
flowInactivityDuration = try container.decode(
|
||||
SyncedField<FlowInactivityDuration>.self,
|
||||
forKey: .flowInactivityDuration
|
||||
@@ -184,6 +196,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable {
|
||||
activePolishStyleId.updatedAt,
|
||||
llmThinkingEnabled.updatedAt,
|
||||
flowSkipAppSwitch.updatedAt,
|
||||
flowKeepAliveMode.updatedAt,
|
||||
flowInactivityDuration.updatedAt,
|
||||
].max() ?? .distantPast
|
||||
}
|
||||
@@ -222,6 +235,7 @@ public extension SyncedAppSettingsV2 {
|
||||
activePolishStyleId: field(configuration.activePolishStyleId),
|
||||
llmThinkingEnabled: field(configuration.llmThinkingEnabled),
|
||||
flowSkipAppSwitch: field(configuration.flowSkipAppSwitch),
|
||||
flowKeepAliveMode: field(configuration.flowKeepAliveMode),
|
||||
flowInactivityDuration: field(configuration.flowInactivityDuration)
|
||||
)
|
||||
}
|
||||
@@ -252,6 +266,7 @@ public extension SyncedAppSettingsV2 {
|
||||
activePolishStyleId: field(PolishStylePackCatalog.defaultID),
|
||||
llmThinkingEnabled: field(false),
|
||||
flowSkipAppSwitch: field(legacy.flowSkipAppSwitch),
|
||||
flowKeepAliveMode: field(.liveActivity),
|
||||
flowInactivityDuration: field(legacy.flowInactivityDuration)
|
||||
)
|
||||
}
|
||||
@@ -288,6 +303,7 @@ public extension SyncedAppSettingsV2 {
|
||||
),
|
||||
llmThinkingEnabled: .merge(local: local.llmThinkingEnabled, remote: remote.llmThinkingEnabled),
|
||||
flowSkipAppSwitch: .merge(local: local.flowSkipAppSwitch, remote: remote.flowSkipAppSwitch),
|
||||
flowKeepAliveMode: .merge(local: local.flowKeepAliveMode, remote: remote.flowKeepAliveMode),
|
||||
flowInactivityDuration: .merge(
|
||||
local: local.flowInactivityDuration,
|
||||
remote: remote.flowInactivityDuration
|
||||
@@ -314,6 +330,7 @@ public extension SyncedAppSettingsV2 {
|
||||
configuration.activePolishStyleId = activePolishStyleId.value
|
||||
configuration.llmThinkingEnabled = llmThinkingEnabled.value
|
||||
configuration.flowSkipAppSwitch = flowSkipAppSwitch.value
|
||||
configuration.flowKeepAliveMode = flowKeepAliveMode.value
|
||||
configuration.flowInactivityDuration = flowInactivityDuration.value
|
||||
}
|
||||
|
||||
@@ -342,6 +359,7 @@ public extension SyncedAppSettingsV2 {
|
||||
patch(©.activePolishStyleId, value: configuration.activePolishStyleId)
|
||||
patch(©.llmThinkingEnabled, value: configuration.llmThinkingEnabled)
|
||||
patch(©.flowSkipAppSwitch, value: configuration.flowSkipAppSwitch)
|
||||
patch(©.flowKeepAliveMode, value: configuration.flowKeepAliveMode)
|
||||
patch(©.flowInactivityDuration, value: configuration.flowInactivityDuration)
|
||||
return copy
|
||||
}
|
||||
@@ -373,6 +391,7 @@ public extension SyncedAppSettingsV2 {
|
||||
touch(©.activePolishStyleId, value: configuration.activePolishStyleId)
|
||||
touch(©.llmThinkingEnabled, value: configuration.llmThinkingEnabled)
|
||||
touch(©.flowSkipAppSwitch, value: configuration.flowSkipAppSwitch)
|
||||
touch(©.flowKeepAliveMode, value: configuration.flowKeepAliveMode)
|
||||
touch(©.flowInactivityDuration, value: configuration.flowInactivityDuration)
|
||||
return copy
|
||||
}
|
||||
|
||||
@@ -312,11 +312,22 @@ public enum FlowSessionBridge {
|
||||
defaults: UserDefaults? = nil
|
||||
) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration(defaults: store)
|
||||
if FlowSessionPolicy.usesInactivityExpiry(defaults: store) {
|
||||
markSessionActiveWithExpiry(duration: duration, sessionId: sessionId, defaults: store)
|
||||
} else {
|
||||
markSessionActivePersistent(sessionId: sessionId, defaults: store)
|
||||
}
|
||||
}
|
||||
|
||||
/// PiP keep-alive: session stays valid until explicit teardown (no idle expiry).
|
||||
public static func markSessionActivePersistent(
|
||||
sessionId: UUID? = nil,
|
||||
defaults: UserDefaults? = nil
|
||||
) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
let now = Date().timeIntervalSince1970
|
||||
let expires = now + resolvedDuration
|
||||
store.set(true, forKey: FlowSessionKeys.flowSessionActive)
|
||||
store.set(expires, forKey: FlowSessionKeys.flowSessionExpires)
|
||||
store.removeObject(forKey: FlowSessionKeys.flowSessionExpires)
|
||||
store.set(now, forKey: FlowSessionKeys.lastActivityAt)
|
||||
writeHeartbeat(defaults: store)
|
||||
clearTranscription(defaults: store)
|
||||
@@ -331,7 +342,7 @@ public enum FlowSessionBridge {
|
||||
heartbeatAt: now,
|
||||
engineMode: AppGroupConfiguration.load(fromAvailable: store).engineMode,
|
||||
localeId: AppGroupConfiguration.load(fromAvailable: store).localeId,
|
||||
sessionExpiresAt: expires,
|
||||
sessionExpiresAt: nil,
|
||||
hostGeneration: store.string(forKey: FlowSessionKeys.hostGeneration)
|
||||
)
|
||||
if let data = encode(snapshot) {
|
||||
@@ -343,6 +354,42 @@ public enum FlowSessionBridge {
|
||||
flush(store)
|
||||
}
|
||||
|
||||
private static func markSessionActiveWithExpiry(
|
||||
duration: TimeInterval? = nil,
|
||||
sessionId: UUID? = nil,
|
||||
defaults: UserDefaults
|
||||
) {
|
||||
let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration(defaults: defaults)
|
||||
let now = Date().timeIntervalSince1970
|
||||
let expires = now + resolvedDuration
|
||||
defaults.set(true, forKey: FlowSessionKeys.flowSessionActive)
|
||||
defaults.set(expires, forKey: FlowSessionKeys.flowSessionExpires)
|
||||
defaults.set(now, forKey: FlowSessionKeys.lastActivityAt)
|
||||
writeHeartbeat(defaults: defaults)
|
||||
clearTranscription(defaults: defaults)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.flowCommandPayload)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.flowResultPayload)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.flowAckPayload)
|
||||
if let sessionId {
|
||||
let snapshot = FlowReadySnapshot(
|
||||
sessionId: sessionId,
|
||||
ready: false,
|
||||
reason: .starting,
|
||||
heartbeatAt: now,
|
||||
engineMode: AppGroupConfiguration.load(fromAvailable: defaults).engineMode,
|
||||
localeId: AppGroupConfiguration.load(fromAvailable: defaults).localeId,
|
||||
sessionExpiresAt: expires,
|
||||
hostGeneration: defaults.string(forKey: FlowSessionKeys.hostGeneration)
|
||||
)
|
||||
if let data = encode(snapshot) {
|
||||
defaults.set(data, forKey: FlowSessionKeys.flowReadyPayload)
|
||||
}
|
||||
} else {
|
||||
defaults.removeObject(forKey: FlowSessionKeys.flowReadyPayload)
|
||||
}
|
||||
flush(defaults)
|
||||
}
|
||||
|
||||
public static func markSessionInactive(defaults: UserDefaults? = nil) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.set(false, forKey: FlowSessionKeys.flowSessionActive)
|
||||
@@ -372,6 +419,7 @@ public enum FlowSessionBridge {
|
||||
defaults: UserDefaults? = nil
|
||||
) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
guard FlowSessionPolicy.usesInactivityExpiry(defaults: store) else { return }
|
||||
let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration(defaults: store)
|
||||
let expires = Date().timeIntervalSince1970 + resolvedDuration
|
||||
store.set(true, forKey: FlowSessionKeys.flowSessionActive)
|
||||
@@ -382,6 +430,7 @@ public enum FlowSessionBridge {
|
||||
/// Resets the inactivity timer after utterance completion or explicit activity.
|
||||
public static func touchLastActivity(defaults: UserDefaults? = nil) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
guard FlowSessionPolicy.usesInactivityExpiry(defaults: store) else { return }
|
||||
let now = Date().timeIntervalSince1970
|
||||
let duration = FlowSessionPolicy.sessionDuration(defaults: store)
|
||||
store.set(now, forKey: FlowSessionKeys.lastActivityAt)
|
||||
@@ -419,6 +468,10 @@ public enum FlowSessionBridge {
|
||||
let store = resolvedDefaults(defaults)
|
||||
guard store.bool(forKey: FlowSessionKeys.flowSessionActive) else { return false }
|
||||
|
||||
if !FlowSessionPolicy.usesInactivityExpiry(defaults: store) {
|
||||
return true
|
||||
}
|
||||
|
||||
let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires)
|
||||
return expires > Date().timeIntervalSince1970
|
||||
}
|
||||
|
||||
@@ -25,6 +25,18 @@ public enum FlowSessionPolicy {
|
||||
inactivityDuration(defaults: defaults).timeInterval
|
||||
}
|
||||
|
||||
public static func keepAliveMode(defaults: UserDefaults? = nil) -> FlowKeepAliveMode {
|
||||
let store = resolvedDefaults(defaults)
|
||||
return FlowKeepAliveMode.fromStored(
|
||||
store.string(forKey: AppGroupConfiguration.Keys.flowKeepAliveMode)
|
||||
)
|
||||
}
|
||||
|
||||
/// PiP sessions have no inactivity expiry; only the Live Activity path times out.
|
||||
public static func usesInactivityExpiry(defaults: UserDefaults? = nil) -> Bool {
|
||||
keepAliveMode(defaults: defaults) == .liveActivity
|
||||
}
|
||||
|
||||
private static func resolvedDefaults(_ defaults: UserDefaults?) -> UserDefaults {
|
||||
if let defaults { return defaults }
|
||||
guard let available = AppGroup.defaultsIfAvailable else {
|
||||
|
||||
@@ -33,6 +33,7 @@ final class AppGroupConfigurationTests: XCTestCase {
|
||||
XCTAssertEqual(config.polishIntensity, .default)
|
||||
XCTAssertTrue(config.personalDictionary.entries.isEmpty)
|
||||
XCTAssertTrue(config.flowSkipAppSwitch)
|
||||
XCTAssertEqual(config.flowKeepAliveMode, .liveActivity)
|
||||
XCTAssertEqual(config.flowInactivityDuration, .fiveMinutes)
|
||||
}
|
||||
|
||||
@@ -58,6 +59,7 @@ final class AppGroupConfigurationTests: XCTestCase {
|
||||
config.cursorDragNavigationEnabled = false
|
||||
config.polishIntensity = .light
|
||||
config.flowSkipAppSwitch = false
|
||||
config.flowKeepAliveMode = .pictureInPicture
|
||||
// Use a non-default value so the round-trip actually proves persistence.
|
||||
config.flowInactivityDuration = .threeHours
|
||||
config.save(to: defaults)
|
||||
@@ -81,6 +83,7 @@ final class AppGroupConfigurationTests: XCTestCase {
|
||||
XCTAssertFalse(loaded.cursorDragNavigationEnabled)
|
||||
XCTAssertEqual(loaded.polishIntensity, .light)
|
||||
XCTAssertFalse(loaded.flowSkipAppSwitch)
|
||||
XCTAssertEqual(loaded.flowKeepAliveMode, .pictureInPicture)
|
||||
XCTAssertEqual(loaded.flowInactivityDuration, .threeHours)
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,25 @@ final class FlowSessionPolicyTests: XCTestCase {
|
||||
XCTAssertEqual(FlowInactivityDuration.tenMinutes.timeInterval, 10 * 60)
|
||||
}
|
||||
|
||||
func testKeepAliveModeDefaultsToLiveActivity() {
|
||||
let defaults = makeDefaults()
|
||||
XCTAssertEqual(FlowSessionPolicy.keepAliveMode(defaults: defaults), .liveActivity)
|
||||
XCTAssertTrue(FlowSessionPolicy.usesInactivityExpiry(defaults: defaults))
|
||||
}
|
||||
|
||||
func testPiPSessionHasNoInactivityExpiry() {
|
||||
let defaults = makeDefaults()
|
||||
defaults.set(FlowKeepAliveMode.pictureInPicture.rawValue,
|
||||
forKey: AppGroupConfiguration.Keys.flowKeepAliveMode)
|
||||
FlowSessionBridge.markSessionActive(sessionId: UUID(), defaults: defaults)
|
||||
|
||||
XCTAssertTrue(FlowSessionBridge.isSessionActive(defaults: defaults))
|
||||
XCTAssertNil(FlowSessionBridge.sessionExpiresAt(defaults: defaults))
|
||||
|
||||
FlowSessionBridge.touchLastActivity(defaults: defaults)
|
||||
XCTAssertNil(FlowSessionBridge.sessionExpiresAt(defaults: defaults))
|
||||
}
|
||||
|
||||
func testTouchLastActivityExtendsExpiry() {
|
||||
let defaults = makeDefaults()
|
||||
defaults.set(FlowInactivityDuration.tenMinutes.rawValue, forKey: AppGroupConfiguration.Keys.flowInactivityDuration)
|
||||
|
||||
@@ -63,6 +63,7 @@ final class SettingsCloudSyncTests: XCTestCase {
|
||||
activePolishStyleId: SyncedField(value: "builtin.light", updatedAt: stampA, deviceID: deviceA),
|
||||
llmThinkingEnabled: SyncedField(value: false, updatedAt: stampA, deviceID: deviceA),
|
||||
flowSkipAppSwitch: SyncedField(value: true, updatedAt: stampA, deviceID: deviceA),
|
||||
flowKeepAliveMode: SyncedField(value: .liveActivity, updatedAt: stampA, deviceID: deviceA),
|
||||
flowInactivityDuration: SyncedField(value: .twelveHours, updatedAt: stampA, deviceID: deviceA)
|
||||
)
|
||||
let remote = SyncedAppSettingsV2(
|
||||
@@ -84,6 +85,7 @@ final class SettingsCloudSyncTests: XCTestCase {
|
||||
activePolishStyleId: SyncedField(value: "builtin.formal", updatedAt: stampB, deviceID: deviceB),
|
||||
llmThinkingEnabled: SyncedField(value: true, updatedAt: stampB, deviceID: deviceB),
|
||||
flowSkipAppSwitch: SyncedField(value: false, updatedAt: stampB, deviceID: deviceB),
|
||||
flowKeepAliveMode: SyncedField(value: .pictureInPicture, updatedAt: stampB, deviceID: deviceB),
|
||||
flowInactivityDuration: SyncedField(value: .threeHours, updatedAt: stampB, deviceID: deviceB)
|
||||
)
|
||||
|
||||
|
||||
@@ -133,6 +133,7 @@ targets:
|
||||
NSSpeechRecognitionUsageDescription: "OSGKeyboard uses speech recognition to transcribe your voice. Audio is processed on-device by default, or sent to your configured speech provider only when you enable cloud recognition."
|
||||
UIBackgroundModes:
|
||||
- audio
|
||||
- picture-in-picture
|
||||
NSSupportsLiveActivities: true
|
||||
NSAppTransportSecurity:
|
||||
NSAllowsArbitraryLoads: false
|
||||
|
||||
Reference in New Issue
Block a user