From 70ba3a4359bef002d98f0280e219aa7a1f8262b6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 26 Jul 2026 10:34:52 +0000 Subject: [PATCH] feat(flow): add Picture in Picture keep-alive mode - Add FlowKeepAliveMode (Dynamic Island vs PiP) with iCloud sync - Settings: keep-alive picker; inactivity timeout only for Live Activity - PiP: waveform sample-buffer controller, mic released between utterances - PiP sessions have no idle expiry; user closing PiP ends the session - FlowSessionManager branches hostReady, session start, and utterance paths - UIBackgroundModes picture-in-picture; bilingual strings and tests Co-authored-by: Rocky --- CHANGELOG.md | 1 + .../FlowPictureInPictureController.swift | 307 ++++++++++++++++++ OSGKeyboard/Services/FlowSessionManager.swift | 192 ++++++++++- OSGKeyboard/Views/FlowPiPHostView.swift | 23 ++ OSGKeyboard/Views/MainAppRoot.swift | 8 + OSGKeyboard/Views/SettingsView.swift | 105 +++++- OSGKeyboard/en.lproj/Localizable.strings | 9 + OSGKeyboard/zh-Hans.lproj/Localizable.strings | 9 + .../Models/AppGroupConfiguration.swift | 8 + .../Models/FlowKeepAliveMode.swift | 39 +++ OSGKeyboardShared/Models/ProviderConfig.swift | 19 +- .../Models/SyncedAppSettingsV2.swift | 19 ++ .../Services/FlowSessionBridge.swift | 61 +++- .../Services/FlowSessionPolicy.swift | 12 + .../AppGroupConfigurationTests.swift | 3 + OSGKeyboardTests/FlowSessionPolicyTests.swift | 19 ++ OSGKeyboardTests/SettingsCloudSyncTests.swift | 2 + project.yml | 1 + 18 files changed, 801 insertions(+), 36 deletions(-) create mode 100644 OSGKeyboard/Services/FlowPictureInPictureController.swift create mode 100644 OSGKeyboard/Views/FlowPiPHostView.swift create mode 100644 OSGKeyboardShared/Models/FlowKeepAliveMode.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index d831b18..da0c326 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- **PiP Flow keep-alive**: Settings → Voice session lets you choose **Dynamic Island** (default, unchanged behaviour) or **Picture in Picture** — a live waveform PiP keeps the host alive with the mic released between utterances; closing PiP ends the session. / **PiP Flow 保活**：设置 → 语音会话可选 **灵动岛**（默认，行为不变）或 **画中画** — 实时波形 PiP 保活、句间释麦；关闭 PiP 即结束会话。 - **Mac MLX streaming ASR**: local dictation uses Qwen3-ASR via mlx-audio-swift with overlay partial preview, tail drain, vocabulary prompt, and polish-before-insert. / **Mac MLX 流式 ASR**：本地听写改用 mlx-audio-swift 的 Qwen3-ASR，支持浮层 partial 预览、尾部截断、词库 prompt 与润色后再插入。 ### Changed diff --git a/OSGKeyboard/Services/FlowPictureInPictureController.swift b/OSGKeyboard/Services/FlowPictureInPictureController.swift new file mode 100644 index 0000000..89d0a4f --- /dev/null +++ b/OSGKeyboard/Services/FlowPictureInPictureController.swift @@ -0,0 +1,307 @@ +// FlowPictureInPictureController.swift +// OSGKeyboard · Main App +// +// PiP keep-alive for Flow sessions: enqueues live waveform sample buffers +// so the host process stays eligible for multitasking while the mic is off +// between utterances. + +import AVFoundation +import AVKit +import CoreMedia +import UIKit + +@MainActor +final class FlowPictureInPictureController: NSObject { + /// User closed the PiP window — host should end the Flow session. + var onUserDismissed: (() -> Void)? + + private(set) var isPictureInPictureActive = false + + let displayLayer = AVSampleBufferDisplayLayer() + + private var pipController: AVPictureInPictureController? + private var displayLink: CADisplayLink? + private weak var hostView: UIView? + private var waveformLevels: [Float] = Array(repeating: 0, count: 24) + private var isStoppingProgrammatically = false + private var frameIndex: Int64 = 0 + + // MARK: - Host view + + func attachHostView(_ view: UIView) { + hostView = view + displayLayer.frame = view.bounds + displayLayer.videoGravity = .resizeAspectFill + displayLayer.removeFromSuperlayer() + view.layer.addSublayer(displayLayer) + configureControllerIfNeeded() + } + + func updateHostLayoutIfNeeded() { + guard let hostView else { return } + displayLayer.frame = hostView.bounds + } + + // MARK: - Lifecycle + + @discardableResult + func start() -> Bool { + guard AVPictureInPictureController.isPictureInPictureSupported() else { + return false + } + configureControllerIfNeeded() + startFramePump() + guard pipController != nil else { return false } + + if pipController?.isPictureInPictureActive == true { + isPictureInPictureActive = true + return true + } + + enqueueWaveformFrame() + pipController?.startPictureInPicture() + return true + } + + func startAndWait(timeout: TimeInterval = 4) async -> Bool { + if isPictureInPictureActive { return true } + guard start() else { return false } + + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if isPictureInPictureActive { return true } + if pipController?.isPictureInPictureActive == true { + isPictureInPictureActive = true + return true + } + try? await Task.sleep(nanoseconds: 50_000_000) + } + return isPictureInPictureActive + } + + func stop() { + isStoppingProgrammatically = true + stopFramePump() + pipController?.stopPictureInPicture() + displayLayer.flushAndRemoveImage() + isPictureInPictureActive = false + isStoppingProgrammatically = false + } + + func updateWaveformLevels(_ levels: [Float]) { + guard !levels.isEmpty else { return } + waveformLevels = levels + } + + // MARK: - Private + + private func configureControllerIfNeeded() { + guard pipController == nil else { return } + guard AVPictureInPictureController.isPictureInPictureSupported() else { return } + + let contentSource = AVPictureInPictureController.ContentSource( + sampleBufferDisplayLayer: displayLayer, + playbackDelegate: self + ) + let controller = AVPictureInPictureController(contentSource: contentSource) + controller.delegate = self + controller.canStartPictureInPictureAutomaticallyFromInline = true + pipController = controller + } + + private func startFramePump() { + guard displayLink == nil else { return } + let link = CADisplayLink(target: self, selector: #selector(handleDisplayLink(_:))) + link.preferredFrameRateRange = CAFrameRateRange(minimum: 20, maximum: 30, preferred: 24) + link.add(to: .main, forMode: .common) + displayLink = link + } + + private func stopFramePump() { + displayLink?.invalidate() + displayLink = nil + } + + @objc private func handleDisplayLink(_ link: CADisplayLink) { + enqueueWaveformFrame() + updateHostLayoutIfNeeded() + + if let pipController, !pipController.isPictureInPictureActive, pipController.isPictureInPicturePossible { + pipController.startPictureInPicture() + } + } + + private func enqueueWaveformFrame() { + guard let sampleBuffer = makeWaveformSampleBuffer(levels: resolvedLevels()) else { return } + if displayLayer.status == .failed { + displayLayer.flush() + } + displayLayer.enqueue(sampleBuffer) + } + + private func resolvedLevels() -> [Float] { + if waveformLevels.contains(where: { $0 > 0.02 }) { + return waveformLevels + } + // Idle breathing animation between utterances. + let phase = Float(frameIndex) * 0.12 + return (0.. CMSampleBuffer? { + let width = 320 + let height = 180 + frameIndex += 1 + + var pixelBuffer: CVPixelBuffer? + let attrs: [String: Any] = [ + kCVPixelBufferCGImageCompatibilityKey as String: true, + kCVPixelBufferCGBitmapContextCompatibilityKey as String: true, + ] + let status = CVPixelBufferCreate( + kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + attrs as CFDictionary, + &pixelBuffer + ) + guard status == kCVReturnSuccess, let pixelBuffer else { return nil } + + CVPixelBufferLockBaseAddress(pixelBuffer, []) + defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) } + + guard let base = CVPixelBufferGetBaseAddress(pixelBuffer) else { return nil } + let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer) + let colorSpace = CGColorSpaceCreateDeviceRGB() + guard let context = CGContext( + data: base, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: bytesPerRow, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue + ) else { return nil } + + // Dark backdrop + accent waveform bars. + context.setFillColor(UIColor(red: 0.07, green: 0.09, blue: 0.11, alpha: 1).cgColor) + context.fill(CGRect(x: 0, y: 0, width: width, height: height)) + + let barCount = max(levels.count, 1) + let barWidth = CGFloat(width) / CGFloat(barCount * 2) + let accent = UIColor(red: 0.20, green: 0.78, blue: 0.55, alpha: 1) + context.setFillColor(accent.cgColor) + + for (index, level) in levels.enumerated() { + let clamped = CGFloat(min(max(level, 0), 1)) + let barHeight = max(6, clamped * CGFloat(height) * 0.72) + let x = (CGFloat(index) * 2 + 0.5) * barWidth + let rect = CGRect( + x: x, + y: (CGFloat(height) - barHeight) / 2, + width: barWidth, + height: barHeight + ) + let path = UIBezierPath(roundedRect: rect, cornerRadius: barWidth * 0.35) + context.addPath(path.cgPath) + context.fillPath() + } + + var formatDescription: CMFormatDescription? + CMVideoFormatDescriptionCreateForImageBuffer( + allocator: kCFAllocatorDefault, + imageBuffer: pixelBuffer, + formatDescriptionOut: &formatDescription + ) + guard let formatDescription else { return nil } + + var timing = CMSampleTimingInfo( + duration: CMTime(value: 1, timescale: 24), + presentationTimeStamp: CMTime(value: frameIndex, timescale: 24), + decodeTimeStamp: .invalid + ) + + var sampleBuffer: CMSampleBuffer? + CMSampleBufferCreateForImageBuffer( + allocator: kCFAllocatorDefault, + imageBuffer: pixelBuffer, + dataReady: true, + makeDataReadyCallback: nil, + refcon: nil, + formatDescription: formatDescription, + sampleTiming: &timing, + sampleBufferOut: &sampleBuffer + ) + return sampleBuffer + } +} + +// MARK: - AVPictureInPictureControllerDelegate + +extension FlowPictureInPictureController: AVPictureInPictureControllerDelegate { + func pictureInPictureControllerDidStartPictureInPicture( + _ pictureInPictureController: AVPictureInPictureController + ) { + isPictureInPictureActive = true + } + + func pictureInPictureControllerDidStopPictureInPicture( + _ pictureInPictureController: AVPictureInPictureController + ) { + isPictureInPictureActive = false + stopFramePump() + guard !isStoppingProgrammatically else { return } + onUserDismissed?() + } + + func pictureInPictureController( + _ pictureInPictureController: AVPictureInPictureController, + restoreUserInterfaceForPictureInPictureStopWithCompletionHandler completionHandler: @escaping (Bool) -> Void + ) { + completionHandler(true) + } +} + +// MARK: - AVPictureInPictureSampleBufferPlaybackDelegate + +extension FlowPictureInPictureController: AVPictureInPictureSampleBufferPlaybackDelegate { + func pictureInPictureController( + _ pictureInPictureController: AVPictureInPictureController, + setPlaying playing: Bool + ) { + if playing { + startFramePump() + } else { + stopFramePump() + } + } + + func pictureInPictureControllerTimeRangeForPlayback( + _ pictureInPictureController: AVPictureInPictureController + ) -> CMTimeRange { + CMTimeRange(start: .zero, duration: CMTime(value: 3600, timescale: 1)) + } + + func pictureInPictureControllerIsPlaybackPaused( + _ pictureInPictureController: AVPictureInPictureController + ) -> Bool { + false + } + + func pictureInPictureController( + _ pictureInPictureController: AVPictureInPictureController, + didTransitionToRenderSize newRenderSize: CMVideoDimensions + ) {} + + func pictureInPictureController( + _ pictureInPictureController: AVPictureInPictureController, + skipByInterval skipInterval: CMTime, + completion completionHandler: @escaping () -> Void + ) { + completionHandler() + } +} diff --git a/OSGKeyboard/Services/FlowSessionManager.swift b/OSGKeyboard/Services/FlowSessionManager.swift index 775321d..13546e9 100644 --- a/OSGKeyboard/Services/FlowSessionManager.swift +++ b/OSGKeyboard/Services/FlowSessionManager.swift @@ -23,6 +23,7 @@ final class FlowSessionManager: ObservableObject { @Published var coldStartContext: FlowColdStartContext? private let capture = FlowContinuousCapture() + private let pipController = FlowPictureInPictureController() private let store = AppGroupStore() /// Cloud-engine polish; local engine runs through built-in DeepSeek polish. private let polisher = PolishingService() @@ -78,6 +79,14 @@ final class FlowSessionManager: ObservableObject { private var coldStartRecoveryTask: Task? /// Initial proof window — cold mic sessions often need >2.5s after app switch. private static let coldStartAudioProofTimeout: TimeInterval = 6 + + private var usesPiPKeepAlive: Bool { + FlowSessionPolicy.keepAliveMode() == .pictureInPicture + } + + func attachPiPHostView(_ view: UIView) { + pipController.attachHostView(view) + } /// Guards the once-per-process launch reconciliation (scene reconnects /// recreate the `@StateObject`-owned manager within the same process). private static var didRunLaunchReconciliation = false @@ -122,6 +131,11 @@ final class FlowSessionManager: ObservableObject { kind: .recognitionInterrupted ) } + pipController.onUserDismissed = { [weak self] in + guard let self, self.isActive else { return } + self.debug("PiP dismissed by user — ending Flow session") + self.endSession() + } FlowTerminationCoordinator.register(self) } @@ -276,7 +290,9 @@ final class FlowSessionManager: ObservableObject { // hit the same audio-proof timeout. coldStartRecoveryTask?.cancel() coldStartRecoveryTask = nil - if capture.running { + if usesPiPKeepAlive { + pipController.stop() + } else if capture.running { capture.stop() } sessionASR?.cancel() @@ -330,6 +346,7 @@ final class FlowSessionManager: ObservableObject { if capture.running { capture.stop() } + pipController.stop() endBackgroundKeepAlive() ScreenWakeLock.release() @@ -400,6 +417,7 @@ final class FlowSessionManager: ObservableObject { isUtteranceProcessing = false capture.stop() + pipController.stop() endBackgroundKeepAlive() ScreenWakeLock.release() sessionASR = nil @@ -416,6 +434,10 @@ final class FlowSessionManager: ObservableObject { } func extendSession(duration: TimeInterval? = nil) { + guard !usesPiPKeepAlive else { + refreshHostReady() + return + } let resolved = duration ?? FlowSessionPolicy.sessionDuration() FlowSessionBridge.extendSession(by: resolved) sessionExpiresAt = Date().addingTimeInterval(resolved) @@ -489,6 +511,10 @@ final class FlowSessionManager: ObservableObject { private func reactivateCaptureIfNeeded() async { guard isActive else { return } + if usesPiPKeepAlive, !isUtteranceRecording, !isUtteranceProcessing, !capture.running { + refreshHostReady() + return + } // A system interruption (call / Siri) may be in progress. Probe it: // `setActive(true)` inside `reassertIfRunning` fails while the // interruption is live and succeeds once it ends — which also covers @@ -559,12 +585,22 @@ final class FlowSessionManager: ObservableObject { let pollingAlive = pollingTask != nil && pollingTask?.isCancelled != true let hasRecentAudio = capture.engineHasRecentAudio(maxAge: 2) - let canAcceptUtterance = capture.engineIsLive - && pollingAlive - && hasRecentAudio - && !isUtteranceRecording - && !isUtteranceProcessing - && sessionWarning == nil + let canAcceptUtterance: Bool + if usesPiPKeepAlive { + canAcceptUtterance = pipController.isPictureInPictureActive + && pollingAlive + && !isUtteranceRecording + && !isUtteranceProcessing + && sessionWarning == nil + && !capture.isInterrupted + } else { + canAcceptUtterance = capture.engineIsLive + && pollingAlive + && hasRecentAudio + && !isUtteranceRecording + && !isUtteranceProcessing + && sessionWarning == nil + } let reason: FlowReadySnapshot.Reason if canAcceptUtterance { @@ -575,9 +611,11 @@ final class FlowSessionManager: ObservableObject { reason = .recording } else if isUtteranceProcessing { reason = .processing - } else if !capture.engineIsLive { + } else if usesPiPKeepAlive, !pipController.isPictureInPictureActive { + reason = .starting + } else if !usesPiPKeepAlive, !capture.engineIsLive { reason = .audioEngineNotLive - } else if !hasRecentAudio { + } else if !usesPiPKeepAlive, !hasRecentAudio { reason = .waitingForAudioProof } else { reason = .starting @@ -650,6 +688,11 @@ final class FlowSessionManager: ObservableObject { /// custom keyboard extension sees green immediately. func refreshForInlineKeyboardFocus() async { guard isActive else { return } + if usesPiPKeepAlive { + refreshHostReady() + FlowSessionBridge.writeHeartbeat() + return + } await reactivateCaptureIfNeeded() refreshHostReady() if !FlowSessionBridge.isHostReady() { @@ -662,7 +705,7 @@ final class FlowSessionManager: ObservableObject { /// Extend expiry after utterance completion based on the inactivity policy. private func touchSessionActivity() { - guard isActive else { return } + guard isActive, !usesPiPKeepAlive else { return } FlowSessionBridge.touchLastActivity() if let expires = FlowSessionBridge.sessionExpiresAt() { sessionExpiresAt = Date(timeIntervalSince1970: expires) @@ -693,6 +736,25 @@ final class FlowSessionManager: ObservableObject { return } + if usesPiPKeepAlive { + let pipReady = await pipController.startAndWait() + guard pipReady else { + let message = AppL10n.string("flow.pip.error.unavailable") + sessionWarning = message + traceState("startSessionAsync.failed", extra: "reason=pipUnavailable") + FlowSessionBridge.setHostReady(false) + if isColdStartHandoff { + showColdStartAudioFailure(message: message) + } + debug("PiP keep-alive failed to start") + return + } + activateFlowSessionAfterPiPProof(duration: duration) + traceState("startSessionAsync.ready") + debug("Flow session started (PiP keep-alive), mic released between utterances") + return + } + do { try capture.start() } catch { @@ -729,6 +791,31 @@ final class FlowSessionManager: ObservableObject { debug("Flow session started (\(Int(duration ?? FlowSessionPolicy.sessionDuration()))s inactivity window), continuous capture running") } + private func activateFlowSessionAfterPiPProof(duration: TimeInterval?) { + let sessionId = activeSessionId ?? UUID() + activeSessionId = sessionId + lastHandledCommandSeq = 0 + FlowSessionBridge.markSessionActive(duration: duration, sessionId: sessionId) + FlowSessionDarwin.postSessionChanged() + isActive = true + ScreenWakeLock.acquire() + sessionExpiresAt = nil + + startHeartbeat() + startCommandObserver() + startPolling() + startLevelPublishing() + expiryTask?.cancel() + expiryTask = nil + + bindSessionASRIfNeeded() + scheduleASRWarmup() + FlowLiveActivityController.startSession() + + refreshHostReady() + traceState("activateFlowSessionAfterPiPProof.done") + } + private func activateFlowSessionAfterAudioProof(duration: TimeInterval?) { let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration() let sessionId = activeSessionId ?? UUID() @@ -756,6 +843,12 @@ final class FlowSessionManager: ObservableObject { private func prepareExistingSessionForColdStartReturn() async { guard isColdStartHandoff, isActive else { return } + if usesPiPKeepAlive { + sessionWarning = nil + refreshHostReady() + handleColdStartAfterSessionReady() + return + } await reactivateCaptureIfNeeded() guard await waitForAudioProof() else { let message = AppL10n.string("flow.coldStart.error.audioTimeout") @@ -809,7 +902,7 @@ final class FlowSessionManager: ObservableObject { } private func scheduleAutoReturnToHostIfNeeded(hostEntry: HostAppEntry?) { - let skipSwitch = FlowSessionPolicy.skipAppSwitch() + let skipSwitch = usesPiPKeepAlive || FlowSessionPolicy.skipAppSwitch() guard skipSwitch, hostEntry != nil else { return } Task { @MainActor [weak self] in try? await Task.sleep(nanoseconds: 450_000_000) @@ -829,6 +922,21 @@ final class FlowSessionManager: ObservableObject { coldStartRecoveryTask?.cancel() coldStartRecoveryTask = Task { @MainActor [weak self] in guard let self else { return } + if self.usesPiPKeepAlive { + let recovered = await self.pipController.startAndWait() + self.traceState("coldStartRecovery.pip", extra: "recovered=\(recovered)") + guard !Task.isCancelled, self.isColdStartHandoff else { return } + if recovered { + if self.isActive { + self.refreshHostReady() + self.handleColdStartAfterSessionReady() + } else { + self.activateFlowSessionAfterPiPProof(duration: duration) + self.handleColdStartAfterSessionReady() + } + } + return + } var recovered = false for attempt in 1...3 { guard !Task.isCancelled, self.isColdStartHandoff else { return } @@ -1000,7 +1108,12 @@ final class FlowSessionManager: ObservableObject { switch command.action { case .startRecording: guard !isUtteranceRecording, !isUtteranceProcessing else { return } - beginUtterance(utteranceId: command.utteranceId, commandSeq: command.commandSeq) + Task { @MainActor [weak self] in + await self?.handleStartRecordingCommand( + utteranceId: command.utteranceId, + commandSeq: command.commandSeq + ) + } case .stopRecording: guard currentUtteranceId == command.utteranceId else { return } if isUtteranceRecording { @@ -1070,6 +1183,42 @@ final class FlowSessionManager: ObservableObject { ) } + private func handleStartRecordingCommand(utteranceId: UUID?, commandSeq: Int64) async { + if usesPiPKeepAlive { + refreshHostReady() + let micReady = await ensureCaptureReadyForPiPUtterance() + guard micReady else { + failUtterance( + message: AppL10n.string("flow.coldStart.error.audioTimeout"), + kind: .audioUnavailable + ) + return + } + } + beginUtterance(utteranceId: utteranceId, commandSeq: commandSeq) + } + + private func ensureCaptureReadyForPiPUtterance() async -> Bool { + if capture.engineHasRecentAudio(maxAge: 2) { + return true + } + do { + try capture.start() + } catch { + debug("PiP utterance capture start failed: \(error.localizedDescription)") + return false + } + return await capture.awaitAudioFlowing(timeout: Self.coldStartAudioProofTimeout) + } + + private func releaseCaptureAfterPiPUtteranceIfNeeded() { + guard usesPiPKeepAlive, capture.running else { return } + guard !isUtteranceRecording, !isUtteranceProcessing else { return } + capture.stop() + pipController.updateWaveformLevels([]) + refreshHostReady() + } + private func beginUtterance(utteranceId: UUID? = nil, commandSeq: Int64 = 0) { guard capture.engineHasRecentAudio(maxAge: 2) else { traceState("beginUtterance.blocked", extra: "reason=audioNotRecent") @@ -1207,6 +1356,10 @@ final class FlowSessionManager: ObservableObject { guard let self else { return } let drainReport = await self.capture.endUtteranceAndDrain() FlowDiagnostics.logDrain(drainReport) + if self.usesPiPKeepAlive { + self.capture.stop() + self.pipController.updateWaveformLevels([]) + } await self.finalizeUtterance( sessionId: drainingSessionId, utteranceId: drainingUtteranceId, @@ -1229,6 +1382,7 @@ final class FlowSessionManager: ObservableObject { chunkedPipeline = nil asr.cancel() capture.cancelUtterance() + releaseCaptureAfterPiPUtteranceIfNeeded() currentPartial = "" lastFinal = "" chunkWarnings = [] @@ -1255,6 +1409,7 @@ final class FlowSessionManager: ObservableObject { chunkedPipeline = nil asr.cancel() capture.cancelUtterance() + releaseCaptureAfterPiPUtteranceIfNeeded() currentPartial = "" lastFinal = "" chunkWarnings = [] @@ -1277,6 +1432,8 @@ final class FlowSessionManager: ObservableObject { finalizeTask?.cancel() finalizeTask = nil chunkedPipeline = nil + capture.cancelUtterance() + releaseCaptureAfterPiPUtteranceIfNeeded() currentPartial = "" lastFinal = "" chunkWarnings = [] @@ -1589,6 +1746,9 @@ final class FlowSessionManager: ObservableObject { while !Task.isCancelled { guard let self, self.isActive else { break } let levels = self.capture.currentAudioLevels() + if self.usesPiPKeepAlive { + self.pipController.updateWaveformLevels(levels) + } if levels.contains(where: { $0 > 0 }) { FlowSessionBridge.storeAudioLevels(levels) } @@ -1612,7 +1772,12 @@ final class FlowSessionManager: ObservableObject { while !Task.isCancelled { guard let self else { break } if self.isActive, !self.capture.engineIsLive { - await self.reactivateCaptureIfNeeded() + let shouldReassert = !self.usesPiPKeepAlive + || self.isUtteranceRecording + || self.isUtteranceProcessing + if shouldReassert { + await self.reactivateCaptureIfNeeded() + } } FlowSessionBridge.writeHeartbeat() self.refreshHostReady() @@ -1627,6 +1792,7 @@ final class FlowSessionManager: ObservableObject { } private func scheduleExpiry(after duration: TimeInterval) { + guard !usesPiPKeepAlive else { return } expiryTask?.cancel() expiryTask = Task { @MainActor [weak self] in try? await Task.sleep(nanoseconds: UInt64(duration * 1_000_000_000)) diff --git a/OSGKeyboard/Views/FlowPiPHostView.swift b/OSGKeyboard/Views/FlowPiPHostView.swift new file mode 100644 index 0000000..7d19d8c --- /dev/null +++ b/OSGKeyboard/Views/FlowPiPHostView.swift @@ -0,0 +1,23 @@ +// FlowPiPHostView.swift +// OSGKeyboard · Main App +// +// Hidden host for the PiP sample-buffer display layer (must live in the window hierarchy). + +import SwiftUI +import UIKit + +struct FlowPiPHostView: UIViewRepresentable { + let attach: (UIView) -> Void + + func makeUIView(context: Context) -> UIView { + let view = UIView(frame: CGRect(x: 0, y: 0, width: 2, height: 2)) + view.isUserInteractionEnabled = false + view.backgroundColor = .clear + attach(view) + return view + } + + func updateUIView(_ uiView: UIView, context: Context) { + attach(uiView) + } +} diff --git a/OSGKeyboard/Views/MainAppRoot.swift b/OSGKeyboard/Views/MainAppRoot.swift index 2ae60d2..26c14e9 100644 --- a/OSGKeyboard/Views/MainAppRoot.swift +++ b/OSGKeyboard/Views/MainAppRoot.swift @@ -37,6 +37,14 @@ struct MainAppRoot: View { } } .animation(.easeInOut(duration: 0.2), value: flowManager.coldStartContext != nil) + .background { + FlowPiPHostView { view in + flowManager.attachPiPHostView(view) + } + .frame(width: 2, height: 2) + .opacity(0.001) + .allowsHitTesting(false) + } .onAppear { flowManager.setAppForeground(scenePhase == .active) // Register the URL handler BEFORE the foreground auto-start. diff --git a/OSGKeyboard/Views/SettingsView.swift b/OSGKeyboard/Views/SettingsView.swift index f40ffbe..45577bd 100644 --- a/OSGKeyboard/Views/SettingsView.swift +++ b/OSGKeyboard/Views/SettingsView.swift @@ -29,6 +29,8 @@ struct SettingsView: View { // Dynamic locale list loaded from SFSpeechRecognizer on first appear. @State private var dynamicLocales: [(id: String, onDevice: Bool)] = [] @State private var showResetConfirmation = false + @State private var showActiveFlowSessionAlert = false + @State private var pendingKeepAliveMode: FlowKeepAliveMode? // v0.2.0: no on-device model manager / pending download state — // iOS `SpeechAnalyzer` ships with iOS 26 and needs nothing // downloaded. @@ -105,27 +107,41 @@ struct SettingsView: View { VStack(alignment: .leading, spacing: SettingsListMetrics.sectionLabelSpacing) { sectionHeader("settings.flow.title") VStack(spacing: 0) { - Toggle(isOn: $config.flowSkipAppSwitch) { - VStack(alignment: .leading, spacing: Spacing.xxs) { - Text("settings.flow.skipAppSwitch.title") - .font(TypeStyle.body) - .foregroundStyle(palette.textPrimary) - Text("settings.flow.skipAppSwitch.subtitle") - .font(TypeStyle.caption2) - .foregroundStyle(palette.textTertiary) - } - } - .tint(palette.accent) - .settingsListRow() - - Divider().background(palette.divider) - - FlowInactivityPickerRow( + FlowKeepAliveModePickerRow( selection: Binding( - get: { config.flowInactivityDuration }, - set: { config.flowInactivityDuration = $0 } + get: { config.flowKeepAliveMode }, + set: { newMode in + applyKeepAliveModeChange(newMode) + } ) ) + + if config.flowKeepAliveMode == .liveActivity { + Divider().background(palette.divider) + + FlowInactivityPickerRow( + selection: Binding( + get: { config.flowInactivityDuration }, + set: { config.flowInactivityDuration = $0 } + ) + ) + + Divider().background(palette.divider) + + Toggle(isOn: $config.flowSkipAppSwitch) { + flowSkipAppSwitchLabel + } + .tint(palette.accent) + .settingsListRow() + } else { + Divider().background(palette.divider) + + Text("settings.flow.keepAlive.pictureInPicture.note") + .font(TypeStyle.caption2) + .foregroundStyle(palette.textTertiary) + .frame(maxWidth: .infinity, alignment: .leading) + .settingsListRow() + } } .background(palette.surface, in: RoundedRectangle(cornerRadius: Radius.xl, style: .continuous)) .overlay( @@ -133,6 +149,34 @@ struct SettingsView: View { .stroke(palette.divider, lineWidth: 0.5) ) } + .alert("settings.flow.keepAlive.activeSession.title", isPresented: $showActiveFlowSessionAlert) { + Button("common.done", role: .cancel) { + pendingKeepAliveMode = nil + } + } message: { + Text("settings.flow.keepAlive.activeSession.message") + } + } + + private var flowSkipAppSwitchLabel: some View { + VStack(alignment: .leading, spacing: Spacing.xxs) { + Text("settings.flow.skipAppSwitch.title") + .font(TypeStyle.body) + .foregroundStyle(palette.textPrimary) + Text("settings.flow.skipAppSwitch.subtitle") + .font(TypeStyle.caption2) + .foregroundStyle(palette.textTertiary) + } + } + + private func applyKeepAliveModeChange(_ newMode: FlowKeepAliveMode) { + guard newMode != config.flowKeepAliveMode else { return } + if FlowSessionBridge.isSessionActive() { + pendingKeepAliveMode = newMode + showActiveFlowSessionAlert = true + return + } + config.flowKeepAliveMode = newMode } // MARK: - Engine @@ -520,6 +564,31 @@ private struct AppearancePickerRow: View { } } +// MARK: - Flow keep-alive mode picker row + +private struct FlowKeepAliveModePickerRow: View { + @Binding var selection: FlowKeepAliveMode + + private var options: [(id: String, label: String)] { + FlowKeepAliveMode.allCases.map { mode in + (mode.rawValue, AppL10n.string(mode.labelKey)) + } + } + + var body: some View { + PickerRow( + title: AppL10n.string("settings.flow.keepAlive.title"), + options: options, + selection: Binding( + get: { selection.rawValue }, + set: { newValue in + selection = FlowKeepAliveMode(rawValue: newValue) ?? .liveActivity + } + ) + ) + } +} + // MARK: - Flow inactivity picker row private struct FlowInactivityPickerRow: View { diff --git a/OSGKeyboard/en.lproj/Localizable.strings b/OSGKeyboard/en.lproj/Localizable.strings index 7a01e5b..aff587f 100644 --- a/OSGKeyboard/en.lproj/Localizable.strings +++ b/OSGKeyboard/en.lproj/Localizable.strings @@ -418,6 +418,14 @@ /* Flow session policy */ "settings.flow.title" = "Voice session"; +"settings.flow.keepAlive.title" = "Keep-alive mode"; +"settings.flow.keepAlive.liveActivity" = "Dynamic Island"; +"settings.flow.keepAlive.liveActivity.subtitle" = "Continuous mic session with inactivity timeout."; +"settings.flow.keepAlive.pictureInPicture" = "Picture in Picture"; +"settings.flow.keepAlive.pictureInPicture.subtitle" = "Waveform PiP keeps the app alive; mic is released between utterances."; +"settings.flow.keepAlive.pictureInPicture.note" = "Picture in Picture stays active until you close it. Skip app switch is always on in this mode."; +"settings.flow.keepAlive.activeSession.title" = "End the current session first"; +"settings.flow.keepAlive.activeSession.message" = "Stop the active voice session before changing keep-alive mode."; "settings.flow.skipAppSwitch.title" = "Skip app switch"; "settings.flow.skipAppSwitch.subtitle" = "After a cold start, try to return to the app you came from."; "settings.flow.inactivity.title" = "End session after inactivity"; @@ -438,6 +446,7 @@ "flow.coldStart.permission.title" = "Permission required"; "flow.coldStart.audio.title" = "Voice could not start"; "flow.coldStart.error.audioTimeout" = "The microphone did not become ready in time. It may be busy in another app. Please try again."; +"flow.pip.error.unavailable" = "Picture in Picture could not start. Check that PiP is allowed for OSGKeyboard in Settings."; "flow.coldStart.action.settings" = "Open Settings"; "flow.coldStart.action.retry" = "Try Again"; "flow.coldStart.swipeHint" = "Swipe right along the bottom bar\nto return to your previous app."; diff --git a/OSGKeyboard/zh-Hans.lproj/Localizable.strings b/OSGKeyboard/zh-Hans.lproj/Localizable.strings index 80cc767..948c5cc 100644 --- a/OSGKeyboard/zh-Hans.lproj/Localizable.strings +++ b/OSGKeyboard/zh-Hans.lproj/Localizable.strings @@ -417,6 +417,14 @@ /* Flow 会话策略 */ "settings.flow.title" = "语音会话"; +"settings.flow.keepAlive.title" = "保活方式"; +"settings.flow.keepAlive.liveActivity" = "灵动岛"; +"settings.flow.keepAlive.liveActivity.subtitle" = "麦克风常驻，可按无活动时长结束会话。"; +"settings.flow.keepAlive.pictureInPicture" = "画中画"; +"settings.flow.keepAlive.pictureInPicture.subtitle" = "波形画中画保活；句间释放麦克风。"; +"settings.flow.keepAlive.pictureInPicture.note" = "画中画将持续保活，直到你关闭小窗。此模式下始终跳过应用切换。"; +"settings.flow.keepAlive.activeSession.title" = "请先结束当前会话"; +"settings.flow.keepAlive.activeSession.message" = "更改保活方式前，请先结束正在进行的语音会话。"; "settings.flow.skipAppSwitch.title" = "跳过应用切换"; "settings.flow.skipAppSwitch.subtitle" = "冷启动完成后，尝试自动返回来源 App。"; "settings.flow.inactivity.title" = "无活动后结束会话"; @@ -437,6 +445,7 @@ "flow.coldStart.permission.title" = "需要权限"; "flow.coldStart.audio.title" = "语音暂时无法启动"; "flow.coldStart.error.audioTimeout" = "麦克风未能及时就绪，可能正被通话或其他 App 占用。请稍后重试。"; +"flow.pip.error.unavailable" = "无法启动画中画，请在系统设置中允许 OSGKeyboard 使用画中画。"; "flow.coldStart.action.settings" = "前往设置"; "flow.coldStart.action.retry" = "重试"; "flow.coldStart.swipeHint" = "沿屏幕底部的横条，从左向右滑动\n返回上一个 App。"; diff --git a/OSGKeyboardShared/Models/AppGroupConfiguration.swift b/OSGKeyboardShared/Models/AppGroupConfiguration.swift index 825dcf0..a5938c5 100644 --- a/OSGKeyboardShared/Models/AppGroupConfiguration.swift +++ b/OSGKeyboardShared/Models/AppGroupConfiguration.swift @@ -50,6 +50,8 @@ public struct AppGroupConfiguration: Sendable, Equatable { public static let settingsCloudPayloadV2 = "config.settings.cloudPayload.v2" /// When true, the host app auto-returns to the source app after a cold-start handoff. public static let flowSkipAppSwitch = "config.flowSkipAppSwitch" + /// Raw `FlowKeepAliveMode` value; mutually exclusive PiP vs Live Activity path. + public static let flowKeepAliveMode = "config.flowKeepAliveMode" /// Raw `FlowInactivityDuration` value; session expires after this idle window. public static let flowInactivityDuration = "config.flowInactivityDuration" /// One-shot: remap previous product defaults (30m / 10m) → 5m. @@ -88,6 +90,8 @@ public struct AppGroupConfiguration: Sendable, Equatable { public var settingsICloudSyncEnabled: Bool /// Auto-return to the host app after `startflow` cold start (default on). public var flowSkipAppSwitch: Bool + /// PiP vs Live Activity keep-alive strategy (mutually exclusive). + public var flowKeepAliveMode: FlowKeepAliveMode /// Idle timeout before the Flow session ends; resets on each utterance. public var flowInactivityDuration: FlowInactivityDuration /// Whether local `SpeechAnalyzer` should attach the prepared custom language model. @@ -262,6 +266,9 @@ public struct AppGroupConfiguration: Sendable, Equatable { } return defaults.bool(forKey: Keys.flowSkipAppSwitch) }(), + flowKeepAliveMode: FlowKeepAliveMode.fromStored( + defaults.string(forKey: Keys.flowKeepAliveMode) + ), flowInactivityDuration: FlowInactivityDuration.fromStored( defaults.string(forKey: Keys.flowInactivityDuration) ), @@ -370,6 +377,7 @@ public struct AppGroupConfiguration: Sendable, Equatable { defaults.set(polishIntensity.rawValue, forKey: Keys.polishIntensity) defaults.set(llmThinkingEnabled, forKey: Keys.llmThinkingEnabled) defaults.set(flowSkipAppSwitch, forKey: Keys.flowSkipAppSwitch) + defaults.set(flowKeepAliveMode.rawValue, forKey: Keys.flowKeepAliveMode) defaults.set(flowInactivityDuration.rawValue, forKey: Keys.flowInactivityDuration) defaults.set(localASRCustomLanguageModelEnabled, forKey: Keys.localASRCustomLanguageModelEnabled) defaults.set(personalDictionaryICloudSyncEnabled, forKey: Keys.personalDictionaryICloudSyncEnabled) diff --git a/OSGKeyboardShared/Models/FlowKeepAliveMode.swift b/OSGKeyboardShared/Models/FlowKeepAliveMode.swift new file mode 100644 index 0000000..6b6d9c9 --- /dev/null +++ b/OSGKeyboardShared/Models/FlowKeepAliveMode.swift @@ -0,0 +1,39 @@ +// FlowKeepAliveMode.swift +// OSGKeyboard · Shared +// +// User-selectable Flow session keep-alive strategy (mutually exclusive). + +import Foundation + +public enum FlowKeepAliveMode: String, CaseIterable, Identifiable, Sendable, Codable { + /// Continuous audio capture + Live Activity (current default behaviour). + case liveActivity = "liveActivity" + /// Picture-in-picture waveform keep-alive; mic released between utterances. + case pictureInPicture = "pictureInPicture" + + public var id: String { rawValue } + + /// Existing installs keep the Live Activity / continuous-capture path. + public static let `default`: FlowKeepAliveMode = .liveActivity + + public var labelKey: String { + switch self { + case .liveActivity: return "settings.flow.keepAlive.liveActivity" + case .pictureInPicture: return "settings.flow.keepAlive.pictureInPicture" + } + } + + public var subtitleKey: String { + switch self { + case .liveActivity: return "settings.flow.keepAlive.liveActivity.subtitle" + case .pictureInPicture: return "settings.flow.keepAlive.pictureInPicture.subtitle" + } + } + + public static func fromStored(_ raw: String?) -> FlowKeepAliveMode { + guard let raw, let value = FlowKeepAliveMode(rawValue: raw) else { + return .default + } + return value + } +} diff --git a/OSGKeyboardShared/Models/ProviderConfig.swift b/OSGKeyboardShared/Models/ProviderConfig.swift index cb2ef26..8bd1234 100644 --- a/OSGKeyboardShared/Models/ProviderConfig.swift +++ b/OSGKeyboardShared/Models/ProviderConfig.swift @@ -237,7 +237,22 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable { } } - /// Idle window before an active Flow session expires; resets on each utterance. + /// PiP vs Live Activity keep-alive (mutually exclusive). + @Published public var flowKeepAliveMode: FlowKeepAliveMode { + didSet { + guard !isApplyingConfiguration, flowKeepAliveMode != configuration.flowKeepAliveMode else { return } + configuration.flowKeepAliveMode = flowKeepAliveMode + if flowKeepAliveMode == .pictureInPicture { + configuration.flowSkipAppSwitch = true + if flowSkipAppSwitch != true { + flowSkipAppSwitch = true + } + } + persistConfiguration() + } + } + + /// Idle window before an active Flow session expires; Live Activity mode only. @Published public var flowInactivityDuration: FlowInactivityDuration { didSet { guard !isApplyingConfiguration, @@ -347,6 +362,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable { polishIntensity = configuration.polishIntensity llmThinkingEnabled = configuration.llmThinkingEnabled flowSkipAppSwitch = configuration.flowSkipAppSwitch + flowKeepAliveMode = configuration.flowKeepAliveMode flowInactivityDuration = configuration.flowInactivityDuration localASRCustomLanguageModelEnabled = configuration.localASRCustomLanguageModelEnabled isSyncingProviderAPIKey = true @@ -433,6 +449,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable { polishIntensity = fresh.polishIntensity llmThinkingEnabled = fresh.llmThinkingEnabled flowSkipAppSwitch = fresh.flowSkipAppSwitch + flowKeepAliveMode = fresh.flowKeepAliveMode flowInactivityDuration = fresh.flowInactivityDuration localASRCustomLanguageModelEnabled = fresh.localASRCustomLanguageModelEnabled isSyncingProviderAPIKey = true diff --git a/OSGKeyboardShared/Models/SyncedAppSettingsV2.swift b/OSGKeyboardShared/Models/SyncedAppSettingsV2.swift index 70b928f..ea42260 100644 --- a/OSGKeyboardShared/Models/SyncedAppSettingsV2.swift +++ b/OSGKeyboardShared/Models/SyncedAppSettingsV2.swift @@ -29,6 +29,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable { public var polishIntensity: SyncedField public var llmThinkingEnabled: SyncedField public var flowSkipAppSwitch: SyncedField + public var flowKeepAliveMode: SyncedField public var flowInactivityDuration: SyncedField public init( @@ -50,6 +51,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable { polishIntensity: SyncedField, llmThinkingEnabled: SyncedField, flowSkipAppSwitch: SyncedField, + flowKeepAliveMode: SyncedField, flowInactivityDuration: SyncedField ) { self.schemaVersion = schemaVersion @@ -70,6 +72,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable { self.polishIntensity = polishIntensity self.llmThinkingEnabled = llmThinkingEnabled self.flowSkipAppSwitch = flowSkipAppSwitch + self.flowKeepAliveMode = flowKeepAliveMode self.flowInactivityDuration = flowInactivityDuration } @@ -92,6 +95,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable { case polishIntensity case llmThinkingEnabled case flowSkipAppSwitch + case flowKeepAliveMode case flowInactivityDuration } @@ -124,6 +128,14 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable { forKey: .llmThinkingEnabled ) ?? SyncedField(value: false, updatedAt: polishIntensity.updatedAt, deviceID: polishIntensity.deviceID) flowSkipAppSwitch = try container.decode(SyncedField.self, forKey: .flowSkipAppSwitch) + flowKeepAliveMode = try container.decodeIfPresent( + SyncedField.self, + forKey: .flowKeepAliveMode + ) ?? SyncedField( + value: .liveActivity, + updatedAt: flowSkipAppSwitch.updatedAt, + deviceID: flowSkipAppSwitch.deviceID + ) flowInactivityDuration = try container.decode( SyncedField.self, forKey: .flowInactivityDuration @@ -171,6 +183,7 @@ public struct SyncedAppSettingsV2: Codable, Equatable, Sendable { polishIntensity.updatedAt, llmThinkingEnabled.updatedAt, flowSkipAppSwitch.updatedAt, + flowKeepAliveMode.updatedAt, flowInactivityDuration.updatedAt, ].max() ?? .distantPast } @@ -208,6 +221,7 @@ public extension SyncedAppSettingsV2 { polishIntensity: field(configuration.polishIntensity), llmThinkingEnabled: field(configuration.llmThinkingEnabled), flowSkipAppSwitch: field(configuration.flowSkipAppSwitch), + flowKeepAliveMode: field(configuration.flowKeepAliveMode), flowInactivityDuration: field(configuration.flowInactivityDuration) ) } @@ -237,6 +251,7 @@ public extension SyncedAppSettingsV2 { polishIntensity: field(legacy.polishIntensity), llmThinkingEnabled: field(false), flowSkipAppSwitch: field(legacy.flowSkipAppSwitch), + flowKeepAliveMode: field(.liveActivity), flowInactivityDuration: field(legacy.flowInactivityDuration) ) } @@ -269,6 +284,7 @@ public extension SyncedAppSettingsV2 { polishIntensity: .merge(local: local.polishIntensity, remote: remote.polishIntensity), llmThinkingEnabled: .merge(local: local.llmThinkingEnabled, remote: remote.llmThinkingEnabled), flowSkipAppSwitch: .merge(local: local.flowSkipAppSwitch, remote: remote.flowSkipAppSwitch), + flowKeepAliveMode: .merge(local: local.flowKeepAliveMode, remote: remote.flowKeepAliveMode), flowInactivityDuration: .merge( local: local.flowInactivityDuration, remote: remote.flowInactivityDuration @@ -294,6 +310,7 @@ public extension SyncedAppSettingsV2 { configuration.polishIntensity = polishIntensity.value configuration.llmThinkingEnabled = llmThinkingEnabled.value configuration.flowSkipAppSwitch = flowSkipAppSwitch.value + configuration.flowKeepAliveMode = flowKeepAliveMode.value configuration.flowInactivityDuration = flowInactivityDuration.value } @@ -321,6 +338,7 @@ public extension SyncedAppSettingsV2 { patch(©.polishIntensity, value: configuration.polishIntensity) patch(©.llmThinkingEnabled, value: configuration.llmThinkingEnabled) patch(©.flowSkipAppSwitch, value: configuration.flowSkipAppSwitch) + patch(©.flowKeepAliveMode, value: configuration.flowKeepAliveMode) patch(©.flowInactivityDuration, value: configuration.flowInactivityDuration) return copy } @@ -351,6 +369,7 @@ public extension SyncedAppSettingsV2 { touch(©.polishIntensity, value: configuration.polishIntensity) touch(©.llmThinkingEnabled, value: configuration.llmThinkingEnabled) touch(©.flowSkipAppSwitch, value: configuration.flowSkipAppSwitch) + touch(©.flowKeepAliveMode, value: configuration.flowKeepAliveMode) touch(©.flowInactivityDuration, value: configuration.flowInactivityDuration) return copy } diff --git a/OSGKeyboardShared/Services/FlowSessionBridge.swift b/OSGKeyboardShared/Services/FlowSessionBridge.swift index be3bf3f..364e1eb 100644 --- a/OSGKeyboardShared/Services/FlowSessionBridge.swift +++ b/OSGKeyboardShared/Services/FlowSessionBridge.swift @@ -312,11 +312,22 @@ public enum FlowSessionBridge { defaults: UserDefaults? = nil ) { let store = resolvedDefaults(defaults) - let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration(defaults: store) + if FlowSessionPolicy.usesInactivityExpiry(defaults: store) { + markSessionActiveWithExpiry(duration: duration, sessionId: sessionId, defaults: store) + } else { + markSessionActivePersistent(sessionId: sessionId, defaults: store) + } + } + + /// PiP keep-alive: session stays valid until explicit teardown (no idle expiry). + public static func markSessionActivePersistent( + sessionId: UUID? = nil, + defaults: UserDefaults? = nil + ) { + let store = resolvedDefaults(defaults) let now = Date().timeIntervalSince1970 - let expires = now + resolvedDuration store.set(true, forKey: FlowSessionKeys.flowSessionActive) - store.set(expires, forKey: FlowSessionKeys.flowSessionExpires) + store.removeObject(forKey: FlowSessionKeys.flowSessionExpires) store.set(now, forKey: FlowSessionKeys.lastActivityAt) writeHeartbeat(defaults: store) clearTranscription(defaults: store) @@ -331,7 +342,7 @@ public enum FlowSessionBridge { heartbeatAt: now, engineMode: AppGroupConfiguration.load(fromAvailable: store).engineMode, localeId: AppGroupConfiguration.load(fromAvailable: store).localeId, - sessionExpiresAt: expires, + sessionExpiresAt: nil, hostGeneration: store.string(forKey: FlowSessionKeys.hostGeneration) ) if let data = encode(snapshot) { @@ -343,6 +354,42 @@ public enum FlowSessionBridge { flush(store) } + private static func markSessionActiveWithExpiry( + duration: TimeInterval? = nil, + sessionId: UUID? = nil, + defaults: UserDefaults + ) { + let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration(defaults: defaults) + let now = Date().timeIntervalSince1970 + let expires = now + resolvedDuration + defaults.set(true, forKey: FlowSessionKeys.flowSessionActive) + defaults.set(expires, forKey: FlowSessionKeys.flowSessionExpires) + defaults.set(now, forKey: FlowSessionKeys.lastActivityAt) + writeHeartbeat(defaults: defaults) + clearTranscription(defaults: defaults) + defaults.removeObject(forKey: FlowSessionKeys.flowCommandPayload) + defaults.removeObject(forKey: FlowSessionKeys.flowResultPayload) + defaults.removeObject(forKey: FlowSessionKeys.flowAckPayload) + if let sessionId { + let snapshot = FlowReadySnapshot( + sessionId: sessionId, + ready: false, + reason: .starting, + heartbeatAt: now, + engineMode: AppGroupConfiguration.load(fromAvailable: defaults).engineMode, + localeId: AppGroupConfiguration.load(fromAvailable: defaults).localeId, + sessionExpiresAt: expires, + hostGeneration: defaults.string(forKey: FlowSessionKeys.hostGeneration) + ) + if let data = encode(snapshot) { + defaults.set(data, forKey: FlowSessionKeys.flowReadyPayload) + } + } else { + defaults.removeObject(forKey: FlowSessionKeys.flowReadyPayload) + } + flush(defaults) + } + public static func markSessionInactive(defaults: UserDefaults? = nil) { let store = resolvedDefaults(defaults) store.set(false, forKey: FlowSessionKeys.flowSessionActive) @@ -372,6 +419,7 @@ public enum FlowSessionBridge { defaults: UserDefaults? = nil ) { let store = resolvedDefaults(defaults) + guard FlowSessionPolicy.usesInactivityExpiry(defaults: store) else { return } let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration(defaults: store) let expires = Date().timeIntervalSince1970 + resolvedDuration store.set(true, forKey: FlowSessionKeys.flowSessionActive) @@ -382,6 +430,7 @@ public enum FlowSessionBridge { /// Resets the inactivity timer after utterance completion or explicit activity. public static func touchLastActivity(defaults: UserDefaults? = nil) { let store = resolvedDefaults(defaults) + guard FlowSessionPolicy.usesInactivityExpiry(defaults: store) else { return } let now = Date().timeIntervalSince1970 let duration = FlowSessionPolicy.sessionDuration(defaults: store) store.set(now, forKey: FlowSessionKeys.lastActivityAt) @@ -419,6 +468,10 @@ public enum FlowSessionBridge { let store = resolvedDefaults(defaults) guard store.bool(forKey: FlowSessionKeys.flowSessionActive) else { return false } + if !FlowSessionPolicy.usesInactivityExpiry(defaults: store) { + return true + } + let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires) return expires > Date().timeIntervalSince1970 } diff --git a/OSGKeyboardShared/Services/FlowSessionPolicy.swift b/OSGKeyboardShared/Services/FlowSessionPolicy.swift index 19f24fb..2b27a7d 100644 --- a/OSGKeyboardShared/Services/FlowSessionPolicy.swift +++ b/OSGKeyboardShared/Services/FlowSessionPolicy.swift @@ -25,6 +25,18 @@ public enum FlowSessionPolicy { inactivityDuration(defaults: defaults).timeInterval } + public static func keepAliveMode(defaults: UserDefaults? = nil) -> FlowKeepAliveMode { + let store = resolvedDefaults(defaults) + return FlowKeepAliveMode.fromStored( + store.string(forKey: AppGroupConfiguration.Keys.flowKeepAliveMode) + ) + } + + /// PiP sessions have no inactivity expiry; only the Live Activity path times out. + public static func usesInactivityExpiry(defaults: UserDefaults? = nil) -> Bool { + keepAliveMode(defaults: defaults) == .liveActivity + } + private static func resolvedDefaults(_ defaults: UserDefaults?) -> UserDefaults { if let defaults { return defaults } guard let available = AppGroup.defaultsIfAvailable else { diff --git a/OSGKeyboardTests/AppGroupConfigurationTests.swift b/OSGKeyboardTests/AppGroupConfigurationTests.swift index fcb40d5..3de14a1 100644 --- a/OSGKeyboardTests/AppGroupConfigurationTests.swift +++ b/OSGKeyboardTests/AppGroupConfigurationTests.swift @@ -33,6 +33,7 @@ final class AppGroupConfigurationTests: XCTestCase { XCTAssertEqual(config.polishIntensity, .default) XCTAssertTrue(config.personalDictionary.entries.isEmpty) XCTAssertTrue(config.flowSkipAppSwitch) + XCTAssertEqual(config.flowKeepAliveMode, .liveActivity) XCTAssertEqual(config.flowInactivityDuration, .fiveMinutes) } @@ -58,6 +59,7 @@ final class AppGroupConfigurationTests: XCTestCase { config.cursorDragNavigationEnabled = false config.polishIntensity = .light config.flowSkipAppSwitch = false + config.flowKeepAliveMode = .pictureInPicture // Use a non-default value so the round-trip actually proves persistence. config.flowInactivityDuration = .threeHours config.save(to: defaults) @@ -81,6 +83,7 @@ final class AppGroupConfigurationTests: XCTestCase { XCTAssertFalse(loaded.cursorDragNavigationEnabled) XCTAssertEqual(loaded.polishIntensity, .light) XCTAssertFalse(loaded.flowSkipAppSwitch) + XCTAssertEqual(loaded.flowKeepAliveMode, .pictureInPicture) XCTAssertEqual(loaded.flowInactivityDuration, .threeHours) } diff --git a/OSGKeyboardTests/FlowSessionPolicyTests.swift b/OSGKeyboardTests/FlowSessionPolicyTests.swift index ab46941..c2367f4 100644 --- a/OSGKeyboardTests/FlowSessionPolicyTests.swift +++ b/OSGKeyboardTests/FlowSessionPolicyTests.swift @@ -29,6 +29,25 @@ final class FlowSessionPolicyTests: XCTestCase { XCTAssertEqual(FlowInactivityDuration.tenMinutes.timeInterval, 10 * 60) } + func testKeepAliveModeDefaultsToLiveActivity() { + let defaults = makeDefaults() + XCTAssertEqual(FlowSessionPolicy.keepAliveMode(defaults: defaults), .liveActivity) + XCTAssertTrue(FlowSessionPolicy.usesInactivityExpiry(defaults: defaults)) + } + + func testPiPSessionHasNoInactivityExpiry() { + let defaults = makeDefaults() + defaults.set(FlowKeepAliveMode.pictureInPicture.rawValue, + forKey: AppGroupConfiguration.Keys.flowKeepAliveMode) + FlowSessionBridge.markSessionActive(sessionId: UUID(), defaults: defaults) + + XCTAssertTrue(FlowSessionBridge.isSessionActive(defaults: defaults)) + XCTAssertNil(FlowSessionBridge.sessionExpiresAt(defaults: defaults)) + + FlowSessionBridge.touchLastActivity(defaults: defaults) + XCTAssertNil(FlowSessionBridge.sessionExpiresAt(defaults: defaults)) + } + func testTouchLastActivityExtendsExpiry() { let defaults = makeDefaults() defaults.set(FlowInactivityDuration.tenMinutes.rawValue, forKey: AppGroupConfiguration.Keys.flowInactivityDuration) diff --git a/OSGKeyboardTests/SettingsCloudSyncTests.swift b/OSGKeyboardTests/SettingsCloudSyncTests.swift index 50f12ef..64a51be 100644 --- a/OSGKeyboardTests/SettingsCloudSyncTests.swift +++ b/OSGKeyboardTests/SettingsCloudSyncTests.swift @@ -62,6 +62,7 @@ final class SettingsCloudSyncTests: XCTestCase { polishIntensity: SyncedField(value: .medium, updatedAt: stampA, deviceID: deviceA), llmThinkingEnabled: SyncedField(value: false, updatedAt: stampA, deviceID: deviceA), flowSkipAppSwitch: SyncedField(value: true, updatedAt: stampA, deviceID: deviceA), + flowKeepAliveMode: SyncedField(value: .liveActivity, updatedAt: stampA, deviceID: deviceA), flowInactivityDuration: SyncedField(value: .twelveHours, updatedAt: stampA, deviceID: deviceA) ) let remote = SyncedAppSettingsV2( @@ -82,6 +83,7 @@ final class SettingsCloudSyncTests: XCTestCase { polishIntensity: SyncedField(value: .light, updatedAt: stampB, deviceID: deviceB), llmThinkingEnabled: SyncedField(value: true, updatedAt: stampB, deviceID: deviceB), flowSkipAppSwitch: SyncedField(value: false, updatedAt: stampB, deviceID: deviceB), + flowKeepAliveMode: SyncedField(value: .pictureInPicture, updatedAt: stampB, deviceID: deviceB), flowInactivityDuration: SyncedField(value: .threeHours, updatedAt: stampB, deviceID: deviceB) ) diff --git a/project.yml b/project.yml index 3ec9326..cec442a 100644 --- a/project.yml +++ b/project.yml @@ -133,6 +133,7 @@ targets: NSSpeechRecognitionUsageDescription: "OSGKeyboard uses speech recognition to transcribe your voice. Audio is processed on-device by default, or sent to your configured speech provider only when you enable cloud recognition." UIBackgroundModes: - audio + - picture-in-picture NSSupportsLiveActivities: true NSAppTransportSecurity: NSAllowsArbitraryLoads: false