rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(release): bump version to 1.6.5 (build 51)

Browse Source 
Ship automatic low-profile PiP keep-alive so background dictation stays ready without visible startup UI or idle audio work.
This commit is contained in:
Rocky
2026-08-06 21:39:31 +08:00
parent 9b02e8948c
commit 77504facc8
10 changed files with 193 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OSGKeyboard/Services/FlowPictureInPictureController.swift
+100 -27
View File
@@ -1,9 +1,10 @@
// FlowPictureInPictureController.swift
// OSGKeyboard · Main App
//
// PiP keep-alive for Flow sessions: enqueues a looping “tuck to edge”
// teaching animation (OSG logo card) so the host stays eligible for
// multitasking while the mic is off between utterances.
// Low-profile PiP keep-alive for Flow sessions. Production uses a transparent,
// static AVPictureInPictureVideoCallViewController with a 0.1 pt content
// height, no frame pump, and no idle audio session. The legacy sample-buffer
// teaching animation remains as a code-level fallback.
import AVFoundation
import AVKit
@@ -47,6 +48,8 @@ final class FlowPictureInPictureController: NSObject {
let displayLayer = AVSampleBufferDisplayLayer()
private var pipController: AVPictureInPictureController?
private var videoCallContentController: AVPictureInPictureVideoCallViewController?
private var transparentContentView: UIView?
private var displayLink: CADisplayLink?
private weak var hostView: UIView?
private var isStoppingProgrammatically = false
@@ -56,6 +59,17 @@ final class FlowPictureInPictureController: NSObject {
/// Last system failure reported by the PiP delegate (cleared on each start).
private var lastSystemStartFailure: Error?
/// Production route: a static, transparent VideoCall PiP. Unlike the
/// sample-buffer teaching animation, this needs no video frame pump and can
/// release AVAudioSession after PiP becomes active.
private let usesLowPowerVideoCallPiP = true
/// Community implementations confirm that VideoCall PiP accepts an extreme
/// aspect ratio. The system may clamp its final width, but a 0.1 pt content
/// height makes the surface visually negligible without private positioning
/// APIs.
private static let lowProfileContentSize = CGSize(width: 300, height: 0.1)
private enum Canvas {
static let width = 480
static let height = 270
@@ -69,6 +83,10 @@ final class FlowPictureInPictureController: NSObject {
func attachHostView(_ view: UIView) {
hostView = view
hasHostView = true
if usesLowPowerVideoCallPiP {
displayLayer.removeFromSuperlayer()
return
}
let bounds = view.bounds
displayLayer.frame = (bounds.width >= 1 && bounds.height >= 1)
? bounds
@@ -114,20 +132,25 @@ final class FlowPictureInPictureController: NSObject {
pipController = nil
}
configureControllerIfNeeded()
warmLogoCacheIfNeeded()
animationStartedAt = CACurrentMediaTime()
startFramePump()
if !usesLowPowerVideoCallPiP {
warmLogoCacheIfNeeded()
animationStartedAt = CACurrentMediaTime()
startFramePump()
}
guard pipController != nil else { return false }
if pipController?.isPictureInPictureActive == true {
isPictureInPictureActive = true
releaseAudioSessionForLowPowerPiP()
return true
}
// Prime a few frames before asking the system to start PiP.
enqueueGuideFrame()
enqueueGuideFrame()
pipController?.invalidatePlaybackState()
if !usesLowPowerVideoCallPiP {
// Prime a few frames before asking the system to start PiP.
enqueueGuideFrame()
enqueueGuideFrame()
pipController?.invalidatePlaybackState()
}
pipController?.startPictureInPicture()
return true
}
@@ -200,10 +223,12 @@ final class FlowPictureInPictureController: NSObject {
isStoppingProgrammatically = true
stopFramePump()
pipController?.stopPictureInPicture()
displayLayer.sampleBufferRenderer.flush(
removingDisplayedImage: true,
completionHandler: nil
)
if !usesLowPowerVideoCallPiP {
displayLayer.sampleBufferRenderer.flush(
removingDisplayedImage: true,
completionHandler: nil
)
}
isPictureInPictureActive = false
animationStartedAt = nil
lastSystemStartFailure = nil
@@ -214,13 +239,20 @@ final class FlowPictureInPictureController: NSObject {
/// `canStartPictureInPictureAutomaticallyFromInline` can take over.
func prepareForBackgroundAutoStart() async {
guard isPictureInPictureActive || pipController != nil else { return }
_ = await activateAudioSessionForPiP()
startFramePump()
enqueueGuideFrame()
pipController?.invalidatePlaybackState()
if !isPictureInPictureActive {
pipController?.startPictureInPicture()
if isPictureInPictureActive {
releaseAudioSessionForLowPowerPiP()
return
}
_ = await activateAudioSessionForPiP()
if !usesLowPowerVideoCallPiP {
startFramePump()
enqueueGuideFrame()
pipController?.invalidatePlaybackState()
}
// `canStartPictureInPictureAutomaticallyFromInline` is the primary
// transition when the app backgrounds. This explicit request is a
// foreground/inactive fallback and remains idempotent.
pipController?.startPictureInPicture()
}
/// Keep the shared audio session eligible for PiP after capture stops its
@@ -228,7 +260,11 @@ final class FlowPictureInPictureController: NSObject {
/// the stable playAndRecord category instead of flipping back to playback.
@discardableResult
func reassertKeepAliveAudioSession() async -> Bool {
await activateAudioSessionForPiP()
if usesLowPowerVideoCallPiP, isPictureInPictureActive {
releaseAudioSessionForLowPowerPiP()
return true
}
return await activateAudioSessionForPiP()
}
/// Kept for FlowSessionManager call sites; guide animation ignores live levels.
@@ -267,10 +303,39 @@ final class FlowPictureInPictureController: NSObject {
guard pipController == nil else { return }
guard AVPictureInPictureController.isPictureInPictureSupported() else { return }
let contentSource = AVPictureInPictureController.ContentSource(
sampleBufferDisplayLayer: displayLayer,
playbackDelegate: self
)
let contentSource: AVPictureInPictureController.ContentSource
if usesLowPowerVideoCallPiP, #available(iOS 15.0, *),
let hostView {
let contentController = AVPictureInPictureVideoCallViewController()
contentController.preferredContentSize = Self.lowProfileContentSize
contentController.view.backgroundColor = .clear
contentController.view.isOpaque = false
contentController.view.layer.backgroundColor = UIColor.clear.cgColor
contentController.view.layer.isOpaque = false
contentController.view.clipsToBounds = true
let transparentView = UIView(frame: contentController.view.bounds)
transparentView.autoresizingMask = [.flexibleWidth, .flexibleHeight]
transparentView.backgroundColor = .clear
transparentView.isOpaque = false
transparentView.isUserInteractionEnabled = false
transparentView.layer.backgroundColor = UIColor.clear.cgColor
transparentView.layer.isOpaque = false
transparentView.layer.opacity = 0
contentController.view.addSubview(transparentView)
videoCallContentController = contentController
transparentContentView = transparentView
contentSource = AVPictureInPictureController.ContentSource(
activeVideoCallSourceView: hostView,
contentViewController: contentController
)
} else {
contentSource = AVPictureInPictureController.ContentSource(
sampleBufferDisplayLayer: displayLayer,
playbackDelegate: self
)
}
let controller = AVPictureInPictureController(contentSource: contentSource)
controller.delegate = self
controller.canStartPictureInPictureAutomaticallyFromInline = true
@@ -279,6 +344,13 @@ final class FlowPictureInPictureController: NSObject {
didActivateAudioSessionBeforeController = true
}
private func releaseAudioSessionForLowPowerPiP() {
guard usesLowPowerVideoCallPiP, isPictureInPictureActive else { return }
stopFramePump()
FlowAudioSessionCoordinator.shared.deactivate()
FlowDiagnostics.log("low-profile PiP active — released audio session and frame pump")
}
private func startFramePump() {
guard displayLink == nil else { return }
let link = CADisplayLink(target: self, selector: #selector(handleDisplayLink(_:)))
@@ -572,6 +644,7 @@ extension FlowPictureInPictureController: @preconcurrency AVPictureInPictureCont
) {
isPictureInPictureActive = true
lastSystemStartFailure = nil
releaseAudioSessionForLowPowerPiP()
}
func pictureInPictureControllerDidStopPictureInPicture(
@@ -587,8 +660,8 @@ extension FlowPictureInPictureController: @preconcurrency AVPictureInPictureCont
_ pictureInPictureController: AVPictureInPictureController,
failedToStartPictureInPictureWithError error: Error
) {
// First attempts often fail while the sample-buffer source is still
// warming; keep retrying via the display link / auto-inline path.
// Sample-buffer fallback may need warm-up retries. VideoCall PiP uses
// the automatic-inline path plus the bounded startAndWait fallback.
lastSystemStartFailure = error
FlowDiagnostics.log("PiP start attempt failed (will retry): \(error.localizedDescription)")
}
OSGKeyboard/Services/FlowSessionManager.swift
+29 -33
View File
@@ -309,10 +309,10 @@ final class FlowSessionManager: ObservableObject {
/// Foreground entry for Flow.
///
/// Default is **light**: clear orphaned Live Activities / permission UI, but
/// do **not** start continuous capture. Auto-capture on every foreground was
/// holding the host at ~170–220 MB RSS and jetsamming the keyboard extension
/// before `KVC.init` could run.
/// Default is **light** for audio work: clear orphaned Live Activities /
/// permission UI and automatically arm the low-profile PiP, but do not
/// start capture or ASR. The transparent PiP is intentionally cheap; the
/// 170–220 MB capture/model path still starts only on explicit speech.
///
/// Pass `startCapture: true` for explicit user intent (Home “Start”, etc.).
/// Keyboard mic cold-start uses `osgkeyboard://startflow` → `startSession`.
@@ -349,14 +349,19 @@ final class FlowSessionManager: ObservableObject {
return
}
guard startCapture else {
let shouldAutoArmPiP = keepAliveMode == .pictureInPicture
guard startCapture || shouldAutoArmPiP else {
OSGDiag.log(
"activateOnForeground light — skip capture (keyboard survival) \(OSGDiag.memoryTag())",
category: "flow"
)
return
}
startSession(reason: "activateOnForeground:\(reason)")
startSession(
reason: shouldAutoArmPiP && !startCapture
? "activateOnForeground.autoPiP:\(reason)"
: "activateOnForeground:\(reason)"
)
}
func dismissColdStartOverlay() {
@@ -976,7 +981,9 @@ final class FlowSessionManager: ObservableObject {
FlowSessionBridge.markSessionActive(duration: duration, sessionId: sessionId)
FlowSessionDarwin.postSessionChanged()
isActive = true
ScreenWakeLock.acquire()
// Low-profile PiP is a system-owned keep-alive surface. Keeping the
// display awake wastes far more power than the static PiP itself.
ScreenWakeLock.release()
sessionExpiresAt = nil
startHeartbeat()
@@ -1097,11 +1104,9 @@ final class FlowSessionManager: ObservableObject {
private func presentColdStartReadyOverlay() {
let hostEntry = HostReturnService.pendingHostEntry()
coldStartContext = FlowColdStartContext(
hostEntry: hostEntry,
state: .ready,
keepAliveMode: keepAliveMode
)
// Handoff remains fully automatic; no preparing/ready overlay is
// mounted in the host UI.
coldStartContext = nil
scheduleAutoReturnToHostIfNeeded(hostEntry: hostEntry)
}
@@ -1110,7 +1115,8 @@ final class FlowSessionManager: ObservableObject {
guard skipSwitch, hostEntry != nil else { return }
Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: 450_000_000)
guard let self, self.coldStartContext?.state == .ready else { return }
guard let self, self.isColdStartHandoff, self.isActive,
FlowSessionBridge.isHostReady() else { return }
if HostReturnService.openPendingHostIfPossible() {
self.dismissColdStartOverlay()
}
@@ -1197,38 +1203,24 @@ final class FlowSessionManager: ObservableObject {
}
private func showColdStartPreparing() {
coldStartContext = FlowColdStartContext(
hostEntry: HostReturnService.pendingHostEntry(),
state: .preparing,
keepAliveMode: keepAliveMode
)
coldStartContext = nil
}
private func showColdStartPermissionFailure() {
FlowSessionBridge.setHostReady(false)
coldStartContext = FlowColdStartContext(
hostEntry: HostReturnService.pendingHostEntry(),
state: .failed(.permission(message: permissionWarningMessage())),
keepAliveMode: keepAliveMode
)
coldStartContext = nil
}
private func showColdStartAudioFailure(message: String) {
FlowSessionBridge.setHostReady(false)
coldStartContext = FlowColdStartContext(
hostEntry: HostReturnService.pendingHostEntry(),
state: .failed(.audio(message: message)),
keepAliveMode: keepAliveMode
)
_ = message
coldStartContext = nil
}
private func showColdStartPipFailure(message: String) {
FlowSessionBridge.setHostReady(false)
coldStartContext = FlowColdStartContext(
hostEntry: HostReturnService.pendingHostEntry(),
state: .failed(.pip(message: message)),
keepAliveMode: .pictureInPicture
)
_ = message
coldStartContext = nil
}
private func bindSessionASRIfNeeded(force: Bool = false) {
@@ -2515,6 +2507,10 @@ final class FlowSessionManager: ObservableObject {
levelTask = Task { @MainActor [weak self] in
while !Task.isCancelled {
guard let self, self.isActive else { break }
guard self.isUtteranceRecording || self.capture.engineIsLive else {
try? await Task.sleep(nanoseconds: 1_000_000_000)
continue
}
let levels = self.capture.currentAudioLevels()
if self.usesPiPKeepAlive {
self.pipController.updateWaveformLevels(levels)
OSGKeyboard/Views/MainAppRoot.swift
+2 -15
View File
@@ -30,19 +30,6 @@ struct MainAppRoot: View {
}
.environment(\.locale, config.uiLanguage.swiftUILocale)
.environmentObject(flowManager)
.overlay {
if let context = flowManager.coldStartContext {
FlowColdStartOverlay(
context: context,
onReturnToHost: { flowManager.returnToPendingHostFromColdStart() },
onDismiss: { flowManager.dismissColdStartOverlay() },
onRetry: { flowManager.retryColdStartReadiness() },
onOpenSettings: { flowManager.openColdStartPermissionSettings() }
)
.transition(.opacity)
}
}
.animation(.easeInOut(duration: 0.2), value: flowManager.coldStartContext != nil)
.background {
FlowPiPHostView { view in
flowManager.attachPiPHostView(view)
@@ -75,8 +62,8 @@ struct MainAppRoot: View {
// Heavy work (Flow / CLM / Rime) only after onboarding. Doing it
// earlier jetsams the host (~150 MB+) and the keyboard dies with it.
if config.hasCompletedOnboarding {
// Light path only: never auto-start continuous capture here.
// Capture starts on Home Start / keyboard startflow / mic press.
// Automatically arm the low-profile PiP on every host open.
// Capture/ASR remain lazy and start only on an actual mic press.
flowManager.activateOnForeground(reason: "MainAppRoot.onAppear")
schedulePostOnboardingWarmup(reason: "MainAppRoot.onAppear")
} else {