chore(release): bump version to 1.0.1 (build 27)
Release the current PiP reliability, polish style, macOS dictionary, and UI updates.
This commit is contained in:
@@ -107,6 +107,7 @@
|
||||
<key>UIBackgroundModes</key>
|
||||
<array>
|
||||
<string>audio</string>
|
||||
<string>picture-in-picture</string>
|
||||
</array>
|
||||
<key>UILaunchScreen</key>
|
||||
<dict>
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
// DictionaryAliasGenerator.swift
|
||||
// OSGKeyboard · Main App
|
||||
//
|
||||
// After the user manually adds or edits a personal-dictionary term,
|
||||
// asks the built-in DeepSeek endpoint for common ASR misrecognitions.
|
||||
// Runs only in the main app (Settings) — the keyboard extension reads
|
||||
// the persisted aliases on the next polish / correction call.
|
||||
|
||||
import Foundation
|
||||
import OSGKeyboardShared
|
||||
|
||||
struct DictionaryAliasGenerator: Sendable {
|
||||
private let client: LLMClient?
|
||||
private let timeout: TimeInterval
|
||||
|
||||
init(client: LLMClient? = nil, timeout: TimeInterval = 12) {
|
||||
self.client = client
|
||||
self.timeout = timeout
|
||||
}
|
||||
|
||||
func generateAliases(for term: String) async -> [String] {
|
||||
let trimmed = term.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return [] }
|
||||
|
||||
do {
|
||||
let client = try resolveClient()
|
||||
let prompt = Self.makePrompt(for: trimmed)
|
||||
let raw = try await withThrowingTaskGroup(of: String.self) { group in
|
||||
group.addTask {
|
||||
try await client.polish(trimmed, systemPrompt: prompt)
|
||||
}
|
||||
group.addTask {
|
||||
try await Task.sleep(nanoseconds: UInt64(timeout * 1_000_000_000))
|
||||
throw CancellationError()
|
||||
}
|
||||
let result = try await group.next()!
|
||||
group.cancelAll()
|
||||
return result
|
||||
}
|
||||
return Self.parseAliases(from: raw, excludingTerm: trimmed)
|
||||
} catch {
|
||||
#if DEBUG
|
||||
print("⚠️ [DictionaryAliasGenerator] alias generation failed: \(error)")
|
||||
#endif
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
private func resolveClient() throws -> LLMClient {
|
||||
if let client {
|
||||
return client
|
||||
}
|
||||
guard PreconfiguredKeys.isDeepseekConfigured else {
|
||||
throw LLMError.noAPIKey
|
||||
}
|
||||
let preset = LLMProvider.provider(id: "deepseek")
|
||||
return OpenAICompatibleClient(
|
||||
baseURL: preset.defaultBaseURL,
|
||||
apiKey: PreconfiguredKeys.deepseek,
|
||||
model: preset.defaultModel
|
||||
)
|
||||
}
|
||||
|
||||
private static func makePrompt(for term: String) -> String {
|
||||
"""
|
||||
你是语音识别纠错助手。用户把专有词汇「\(term)」加入了个人词库。
|
||||
请列出该词在中文或英文语音输入时最常见的 3–6 个误识别写法(同音字、近音字、拼音混淆、英文误听等)。
|
||||
不要包含正确词「\(term)」本身。
|
||||
只输出 JSON 字符串数组,例如 ["误识别1","误识别2"]。若无合理别名则输出 []。
|
||||
"""
|
||||
}
|
||||
|
||||
static func parseAliases(from raw: String, excludingTerm term: String) -> [String] {
|
||||
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let jsonSlice = extractJSONArray(from: trimmed) ?? trimmed
|
||||
guard let data = jsonSlice.data(using: .utf8),
|
||||
let decoded = try? JSONDecoder().decode([String].self, from: data)
|
||||
else { return [] }
|
||||
|
||||
let termLower = term.lowercased()
|
||||
var seen = Set<String>()
|
||||
var result: [String] = []
|
||||
for alias in decoded {
|
||||
let cleaned = alias.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !cleaned.isEmpty else { continue }
|
||||
let key = cleaned.lowercased()
|
||||
guard key != termLower, !seen.contains(key) else { continue }
|
||||
seen.insert(key)
|
||||
result.append(cleaned)
|
||||
if result.count >= 6 { break }
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private static func extractJSONArray(from text: String) -> String? {
|
||||
guard let start = text.firstIndex(of: "["),
|
||||
let end = text.lastIndex(of: "]"),
|
||||
start < end
|
||||
else { return nil }
|
||||
return String(text[start...end])
|
||||
}
|
||||
}
|
||||
@@ -1,55 +1,120 @@
|
||||
// FlowPictureInPictureController.swift
|
||||
// OSGKeyboard · Main App
|
||||
//
|
||||
// PiP keep-alive for Flow sessions: enqueues live waveform sample buffers
|
||||
// so the host process stays eligible for multitasking while the mic is off
|
||||
// between utterances.
|
||||
// PiP keep-alive for Flow sessions: enqueues a looping “tuck to edge”
|
||||
// teaching animation (OSG logo card) so the host stays eligible for
|
||||
// multitasking while the mic is off between utterances.
|
||||
|
||||
import AVFoundation
|
||||
import AVKit
|
||||
import CoreMedia
|
||||
import UIKit
|
||||
|
||||
/// Why `startAndWait` could not prove an active PiP window.
|
||||
enum FlowPiPStartFailure: Equatable, Sendable {
|
||||
case unsupported
|
||||
case hostNotReady
|
||||
case notPossible
|
||||
case systemRejected
|
||||
case timedOut
|
||||
|
||||
var localizationKey: String {
|
||||
switch self {
|
||||
case .unsupported: return "flow.pip.error.unsupported"
|
||||
case .hostNotReady: return "flow.pip.error.hostNotReady"
|
||||
case .notPossible: return "flow.pip.error.notPossible"
|
||||
case .systemRejected: return "flow.pip.error.systemRejected"
|
||||
case .timedOut: return "flow.pip.error.timedOut"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum FlowPiPStartOutcome: Equatable, Sendable {
|
||||
case started
|
||||
case failed(FlowPiPStartFailure)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
final class FlowPictureInPictureController: NSObject {
|
||||
/// User closed the PiP window — host should end the Flow session.
|
||||
var onUserDismissed: (() -> Void)?
|
||||
|
||||
private(set) var isPictureInPictureActive = false
|
||||
/// True once a host UIView has been attached (may still be awaiting a window).
|
||||
private(set) var hasHostView = false
|
||||
|
||||
let displayLayer = AVSampleBufferDisplayLayer()
|
||||
|
||||
private var pipController: AVPictureInPictureController?
|
||||
private var displayLink: CADisplayLink?
|
||||
private weak var hostView: UIView?
|
||||
private var waveformLevels: [Float] = Array(repeating: 0, count: 24)
|
||||
private var isStoppingProgrammatically = false
|
||||
private var frameIndex: Int64 = 0
|
||||
private var animationStartedAt: CFTimeInterval?
|
||||
private var cachedLogo: CGImage?
|
||||
/// Last system failure reported by the PiP delegate (cleared on each start).
|
||||
private var lastSystemStartFailure: Error?
|
||||
|
||||
private enum Canvas {
|
||||
static let width = 480
|
||||
static let height = 270
|
||||
static let fps: Int32 = 18
|
||||
/// Full teaching loop length (seconds).
|
||||
static let loopDuration: CFTimeInterval = 4.2
|
||||
}
|
||||
|
||||
// MARK: - Host view
|
||||
|
||||
func attachHostView(_ view: UIView) {
|
||||
hostView = view
|
||||
displayLayer.frame = view.bounds
|
||||
hasHostView = true
|
||||
let bounds = view.bounds
|
||||
displayLayer.frame = (bounds.width >= 1 && bounds.height >= 1)
|
||||
? bounds
|
||||
: CGRect(x: 0, y: 0, width: 64, height: 36)
|
||||
displayLayer.videoGravity = .resizeAspectFill
|
||||
displayLayer.removeFromSuperlayer()
|
||||
view.layer.addSublayer(displayLayer)
|
||||
configureControllerIfNeeded()
|
||||
// Do not create AVPictureInPictureController here — it must be built
|
||||
// only after an active AVAudioSession (see `start()`).
|
||||
}
|
||||
|
||||
func updateHostLayoutIfNeeded() {
|
||||
guard let hostView else { return }
|
||||
displayLayer.frame = hostView.bounds
|
||||
let bounds = hostView.bounds
|
||||
displayLayer.frame = (bounds.width >= 1 && bounds.height >= 1)
|
||||
? bounds
|
||||
: CGRect(x: 0, y: 0, width: 64, height: 36)
|
||||
}
|
||||
|
||||
/// Host layer is in a UIWindow — required before `startPictureInPicture()`.
|
||||
var isHostInWindowHierarchy: Bool {
|
||||
hostView?.window != nil
|
||||
}
|
||||
|
||||
// MARK: - Lifecycle
|
||||
|
||||
@discardableResult
|
||||
func start() -> Bool {
|
||||
lastSystemStartFailure = nil
|
||||
guard AVPictureInPictureController.isPictureInPictureSupported() else {
|
||||
return false
|
||||
}
|
||||
guard hasHostView else { return false }
|
||||
|
||||
// Required before constructing the controller; without an active
|
||||
// session, `isPictureInPicturePossible` stays false forever.
|
||||
guard activateAudioSessionForPiP() else {
|
||||
return false
|
||||
}
|
||||
|
||||
// If a controller was somehow created before audio activation, rebuild.
|
||||
if pipController != nil, !didActivateAudioSessionBeforeController {
|
||||
pipController = nil
|
||||
}
|
||||
configureControllerIfNeeded()
|
||||
warmLogoCacheIfNeeded()
|
||||
animationStartedAt = CACurrentMediaTime()
|
||||
startFramePump()
|
||||
guard pipController != nil else { return false }
|
||||
|
||||
@@ -58,43 +123,149 @@ final class FlowPictureInPictureController: NSObject {
|
||||
return true
|
||||
}
|
||||
|
||||
enqueueWaveformFrame()
|
||||
// Prime a few frames before asking the system to start PiP.
|
||||
enqueueGuideFrame()
|
||||
enqueueGuideFrame()
|
||||
pipController?.invalidatePlaybackState()
|
||||
pipController?.startPictureInPicture()
|
||||
return true
|
||||
}
|
||||
|
||||
func startAndWait(timeout: TimeInterval = 4) async -> Bool {
|
||||
if isPictureInPictureActive { return true }
|
||||
guard start() else { return false }
|
||||
/// Waits until the host is windowed and PiP is actually active.
|
||||
/// Does not treat "armed but inactive" as success — that left sessions
|
||||
/// live while `hostReady` stayed false forever.
|
||||
func startAndWait(
|
||||
hostTimeout: TimeInterval = 3,
|
||||
activeTimeout: TimeInterval = 8
|
||||
) async -> FlowPiPStartOutcome {
|
||||
if isPictureInPictureActive { return .started }
|
||||
|
||||
let deadline = Date().addingTimeInterval(timeout)
|
||||
guard AVPictureInPictureController.isPictureInPictureSupported() else {
|
||||
return .failed(.unsupported)
|
||||
}
|
||||
|
||||
let hostReady = await waitForHostInWindow(timeout: hostTimeout)
|
||||
guard hostReady else {
|
||||
return .failed(.hostNotReady)
|
||||
}
|
||||
|
||||
lastSystemStartFailure = nil
|
||||
guard start() else {
|
||||
stopFramePump()
|
||||
if lastSystemStartFailure != nil {
|
||||
return .failed(.systemRejected)
|
||||
}
|
||||
return .failed(hasHostView ? .notPossible : .hostNotReady)
|
||||
}
|
||||
|
||||
let deadline = Date().addingTimeInterval(activeTimeout)
|
||||
while Date() < deadline {
|
||||
if isPictureInPictureActive { return true }
|
||||
if isPictureInPictureActive { return .started }
|
||||
if pipController?.isPictureInPictureActive == true {
|
||||
isPictureInPictureActive = true
|
||||
return true
|
||||
return .started
|
||||
}
|
||||
if let pipController, pipController.isPictureInPicturePossible {
|
||||
pipController.startPictureInPicture()
|
||||
} else {
|
||||
pipController?.startPictureInPicture()
|
||||
}
|
||||
try? await Task.sleep(nanoseconds: 50_000_000)
|
||||
}
|
||||
return isPictureInPictureActive
|
||||
|
||||
if isPictureInPictureActive { return .started }
|
||||
if pipController?.isPictureInPictureActive == true {
|
||||
isPictureInPictureActive = true
|
||||
return .started
|
||||
}
|
||||
|
||||
// Real failure — tear down so the next retry starts clean.
|
||||
let failure: FlowPiPStartFailure
|
||||
if lastSystemStartFailure != nil {
|
||||
failure = .systemRejected
|
||||
} else if pipController?.isPictureInPicturePossible != true {
|
||||
failure = .notPossible
|
||||
} else {
|
||||
failure = .timedOut
|
||||
}
|
||||
FlowDiagnostics.log(
|
||||
"PiP startAndWait failed: \(failure) possible=\(pipController?.isPictureInPicturePossible == true)"
|
||||
)
|
||||
stop()
|
||||
return .failed(failure)
|
||||
}
|
||||
|
||||
func stop() {
|
||||
isStoppingProgrammatically = true
|
||||
stopFramePump()
|
||||
pipController?.stopPictureInPicture()
|
||||
displayLayer.flushAndRemoveImage()
|
||||
displayLayer.sampleBufferRenderer.flush(
|
||||
removingDisplayedImage: true,
|
||||
completionHandler: nil
|
||||
)
|
||||
isPictureInPictureActive = false
|
||||
animationStartedAt = nil
|
||||
lastSystemStartFailure = nil
|
||||
isStoppingProgrammatically = false
|
||||
}
|
||||
|
||||
/// Nudge the sample-buffer source right before resigning active so
|
||||
/// `canStartPictureInPictureAutomaticallyFromInline` can take over.
|
||||
func prepareForBackgroundAutoStart() {
|
||||
guard isPictureInPictureActive || pipController != nil else { return }
|
||||
_ = activateAudioSessionForPiP()
|
||||
startFramePump()
|
||||
enqueueGuideFrame()
|
||||
pipController?.invalidatePlaybackState()
|
||||
if !isPictureInPictureActive {
|
||||
pipController?.startPictureInPicture()
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-activate the playback session after utterance capture releases the
|
||||
/// mic (`setActive(false)`). Without this, PiP can lose eligibility between
|
||||
/// utterances even though the floating window is still visible.
|
||||
@discardableResult
|
||||
func reassertKeepAliveAudioSession() -> Bool {
|
||||
activateAudioSessionForPiP()
|
||||
}
|
||||
|
||||
/// Kept for FlowSessionManager call sites; guide animation ignores live levels.
|
||||
func updateWaveformLevels(_ levels: [Float]) {
|
||||
guard !levels.isEmpty else { return }
|
||||
waveformLevels = levels
|
||||
_ = levels
|
||||
}
|
||||
|
||||
// MARK: - Private
|
||||
|
||||
/// Set once we successfully activate audio before building the controller.
|
||||
private var didActivateAudioSessionBeforeController = false
|
||||
|
||||
@discardableResult
|
||||
private func activateAudioSessionForPiP() -> Bool {
|
||||
do {
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
// Playback (not record) keeps PiP eligible between utterances without
|
||||
// holding the mic. Utterance capture later switches to playAndRecord.
|
||||
try session.setCategory(.playback, mode: .moviePlayback, options: [.mixWithOthers])
|
||||
try session.setActive(true)
|
||||
FlowDiagnostics.log("PiP audio session active category=playback")
|
||||
return true
|
||||
} catch {
|
||||
FlowDiagnostics.log("PiP audio session failed: \(error.localizedDescription)")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
private func waitForHostInWindow(timeout: TimeInterval) async -> Bool {
|
||||
if isHostInWindowHierarchy { return true }
|
||||
let deadline = Date().addingTimeInterval(timeout)
|
||||
while Date() < deadline {
|
||||
if isHostInWindowHierarchy { return true }
|
||||
try? await Task.sleep(nanoseconds: 50_000_000)
|
||||
}
|
||||
return isHostInWindowHierarchy
|
||||
}
|
||||
|
||||
private func configureControllerIfNeeded() {
|
||||
guard pipController == nil else { return }
|
||||
guard AVPictureInPictureController.isPictureInPictureSupported() else { return }
|
||||
@@ -106,13 +277,19 @@ final class FlowPictureInPictureController: NSObject {
|
||||
let controller = AVPictureInPictureController(contentSource: contentSource)
|
||||
controller.delegate = self
|
||||
controller.canStartPictureInPictureAutomaticallyFromInline = true
|
||||
controller.requiresLinearPlayback = true
|
||||
pipController = controller
|
||||
didActivateAudioSessionBeforeController = true
|
||||
}
|
||||
|
||||
private func startFramePump() {
|
||||
guard displayLink == nil else { return }
|
||||
let link = CADisplayLink(target: self, selector: #selector(handleDisplayLink(_:)))
|
||||
link.preferredFrameRateRange = CAFrameRateRange(minimum: 20, maximum: 30, preferred: 24)
|
||||
link.preferredFrameRateRange = CAFrameRateRange(
|
||||
minimum: 12,
|
||||
maximum: 20,
|
||||
preferred: Float(Canvas.fps)
|
||||
)
|
||||
link.add(to: .main, forMode: .common)
|
||||
displayLink = link
|
||||
}
|
||||
@@ -123,43 +300,52 @@ final class FlowPictureInPictureController: NSObject {
|
||||
}
|
||||
|
||||
@objc private func handleDisplayLink(_ link: CADisplayLink) {
|
||||
enqueueWaveformFrame()
|
||||
enqueueGuideFrame()
|
||||
updateHostLayoutIfNeeded()
|
||||
|
||||
if let pipController, !pipController.isPictureInPictureActive, pipController.isPictureInPicturePossible {
|
||||
pipController.startPictureInPicture()
|
||||
guard let pipController, !pipController.isPictureInPictureActive else { return }
|
||||
if frameIndex % Int64(Canvas.fps) == 0 {
|
||||
pipController.invalidatePlaybackState()
|
||||
}
|
||||
// Retry regardless of `isPictureInPicturePossible` — that flag often
|
||||
// lags behind a warm sample-buffer source.
|
||||
pipController.startPictureInPicture()
|
||||
}
|
||||
|
||||
private func enqueueWaveformFrame() {
|
||||
guard let sampleBuffer = makeWaveformSampleBuffer(levels: resolvedLevels()) else { return }
|
||||
if displayLayer.status == .failed {
|
||||
displayLayer.flush()
|
||||
private func enqueueGuideFrame() {
|
||||
guard let sampleBuffer = makeGuideSampleBuffer() else { return }
|
||||
if displayLayer.sampleBufferRenderer.status == .failed {
|
||||
displayLayer.sampleBufferRenderer.flush()
|
||||
}
|
||||
displayLayer.enqueue(sampleBuffer)
|
||||
displayLayer.sampleBufferRenderer.enqueue(sampleBuffer)
|
||||
}
|
||||
|
||||
private func resolvedLevels() -> [Float] {
|
||||
if waveformLevels.contains(where: { $0 > 0.02 }) {
|
||||
return waveformLevels
|
||||
}
|
||||
// Idle breathing animation between utterances.
|
||||
let phase = Float(frameIndex) * 0.12
|
||||
return (0..<waveformLevels.count).map { index in
|
||||
let wave = sin(phase + Float(index) * 0.45)
|
||||
return max(0.04, 0.04 + wave * 0.03)
|
||||
private func warmLogoCacheIfNeeded() {
|
||||
guard cachedLogo == nil else { return }
|
||||
let logoColor = UIColor.white
|
||||
if let brand = UIImage(named: "OSGBrandMark")?
|
||||
.withTintColor(logoColor, renderingMode: .alwaysOriginal)
|
||||
.cgImage {
|
||||
cachedLogo = brand
|
||||
return
|
||||
}
|
||||
cachedLogo = UIImage(named: "osglogo")?
|
||||
.withTintColor(logoColor, renderingMode: .alwaysOriginal)
|
||||
.cgImage
|
||||
}
|
||||
|
||||
private func makeWaveformSampleBuffer(levels: [Float]) -> CMSampleBuffer? {
|
||||
let width = 320
|
||||
let height = 180
|
||||
// MARK: - Frame rendering
|
||||
|
||||
private func makeGuideSampleBuffer() -> CMSampleBuffer? {
|
||||
let width = Canvas.width
|
||||
let height = Canvas.height
|
||||
frameIndex += 1
|
||||
|
||||
var pixelBuffer: CVPixelBuffer?
|
||||
let attrs: [String: Any] = [
|
||||
kCVPixelBufferCGImageCompatibilityKey as String: true,
|
||||
kCVPixelBufferCGBitmapContextCompatibilityKey as String: true,
|
||||
kCVPixelBufferIOSurfacePropertiesKey as String: [:] as [String: Any],
|
||||
]
|
||||
let status = CVPixelBufferCreate(
|
||||
kCFAllocatorDefault,
|
||||
@@ -184,32 +370,17 @@ final class FlowPictureInPictureController: NSObject {
|
||||
bitsPerComponent: 8,
|
||||
bytesPerRow: bytesPerRow,
|
||||
space: colorSpace,
|
||||
// BGRA pixel buffer requires little-endian byte order; without it
|
||||
// R/B channels swap and greens render as purple.
|
||||
bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue
|
||||
| CGBitmapInfo.byteOrder32Little.rawValue
|
||||
) else { return nil }
|
||||
|
||||
// Dark backdrop + accent waveform bars.
|
||||
context.setFillColor(UIColor(red: 0.07, green: 0.09, blue: 0.11, alpha: 1).cgColor)
|
||||
context.fill(CGRect(x: 0, y: 0, width: width, height: height))
|
||||
// Flip to UIKit top-left coordinates for layout math.
|
||||
context.translateBy(x: 0, y: CGFloat(height))
|
||||
context.scaleBy(x: 1, y: -1)
|
||||
|
||||
let barCount = max(levels.count, 1)
|
||||
let barWidth = CGFloat(width) / CGFloat(barCount * 2)
|
||||
let accent = UIColor(red: 0.20, green: 0.78, blue: 0.55, alpha: 1)
|
||||
context.setFillColor(accent.cgColor)
|
||||
|
||||
for (index, level) in levels.enumerated() {
|
||||
let clamped = CGFloat(min(max(level, 0), 1))
|
||||
let barHeight = max(6, clamped * CGFloat(height) * 0.72)
|
||||
let x = (CGFloat(index) * 2 + 0.5) * barWidth
|
||||
let rect = CGRect(
|
||||
x: x,
|
||||
y: (CGFloat(height) - barHeight) / 2,
|
||||
width: barWidth,
|
||||
height: barHeight
|
||||
)
|
||||
let path = UIBezierPath(roundedRect: rect, cornerRadius: barWidth * 0.35)
|
||||
context.addPath(path.cgPath)
|
||||
context.fillPath()
|
||||
}
|
||||
drawGuideFrame(in: context, width: width, height: height)
|
||||
|
||||
var formatDescription: CMFormatDescription?
|
||||
CMVideoFormatDescriptionCreateForImageBuffer(
|
||||
@@ -220,8 +391,8 @@ final class FlowPictureInPictureController: NSObject {
|
||||
guard let formatDescription else { return nil }
|
||||
|
||||
var timing = CMSampleTimingInfo(
|
||||
duration: CMTime(value: 1, timescale: 24),
|
||||
presentationTimeStamp: CMTime(value: frameIndex, timescale: 24),
|
||||
duration: CMTime(value: 1, timescale: Canvas.fps),
|
||||
presentationTimeStamp: CMTime(value: frameIndex, timescale: Canvas.fps),
|
||||
decodeTimeStamp: .invalid
|
||||
)
|
||||
|
||||
@@ -236,17 +407,174 @@ final class FlowPictureInPictureController: NSObject {
|
||||
sampleTiming: &timing,
|
||||
sampleBufferOut: &sampleBuffer
|
||||
)
|
||||
guard let sampleBuffer else { return nil }
|
||||
// Required for sample-buffer PiP sources to present immediately.
|
||||
CMSetAttachment(
|
||||
sampleBuffer,
|
||||
key: kCMSampleAttachmentKey_DisplayImmediately,
|
||||
value: kCFBooleanTrue,
|
||||
attachmentMode: kCMAttachmentMode_ShouldNotPropagate
|
||||
)
|
||||
return sampleBuffer
|
||||
}
|
||||
|
||||
private func drawGuideFrame(in context: CGContext, width: Int, height: Int) {
|
||||
let canvas = CGRect(x: 0, y: 0, width: width, height: height)
|
||||
|
||||
// White PiP backdrop.
|
||||
context.setFillColor(UIColor.white.cgColor)
|
||||
context.fill(canvas)
|
||||
|
||||
// Soft phone silhouette — gives the “screen edge” a visual anchor.
|
||||
let phoneInset = CGFloat(22)
|
||||
let phoneRect = canvas.insetBy(dx: phoneInset, dy: phoneInset)
|
||||
let phonePath = UIBezierPath(roundedRect: phoneRect, cornerRadius: 28)
|
||||
context.setStrokeColor(UIColor(red: 0.898, green: 0.906, blue: 0.922, alpha: 1).cgColor)
|
||||
context.setLineWidth(2.5)
|
||||
context.addPath(phonePath.cgPath)
|
||||
context.strokePath()
|
||||
|
||||
let cardSize = CGSize(width: 148, height: 96)
|
||||
let restOrigin = CGPoint(
|
||||
x: phoneRect.midX - cardSize.width * 0.55,
|
||||
y: phoneRect.midY - cardSize.height * 0.5
|
||||
)
|
||||
// Mostly off the right edge, leaving a peek strip (~28% visible).
|
||||
let tuckedOrigin = CGPoint(
|
||||
x: phoneRect.maxX - cardSize.width * 0.28,
|
||||
y: restOrigin.y
|
||||
)
|
||||
|
||||
let progress = cardTravelProgress()
|
||||
let cardOrigin = CGPoint(
|
||||
x: restOrigin.x + (tuckedOrigin.x - restOrigin.x) * progress,
|
||||
y: restOrigin.y
|
||||
)
|
||||
let cardRect = CGRect(origin: cardOrigin, size: cardSize)
|
||||
|
||||
// Clip so the tucked card disappears past the phone’s right edge.
|
||||
context.saveGState()
|
||||
context.addPath(phonePath.cgPath)
|
||||
context.clip()
|
||||
|
||||
drawLogoCard(in: context, rect: cardRect, tuckProgress: progress)
|
||||
context.restoreGState()
|
||||
}
|
||||
|
||||
private func drawLogoCard(in context: CGContext, rect: CGRect, tuckProgress: CGFloat) {
|
||||
let cardPath = UIBezierPath(roundedRect: rect, cornerRadius: 16)
|
||||
|
||||
context.setFillColor(UIColor(red: 0.20, green: 0.78, blue: 0.55, alpha: 1).cgColor)
|
||||
context.addPath(cardPath.cgPath)
|
||||
context.fillPath()
|
||||
|
||||
context.setStrokeColor(UIColor(red: 0.20, green: 0.78, blue: 0.55, alpha: 1).cgColor)
|
||||
context.setLineWidth(1.5)
|
||||
context.addPath(cardPath.cgPath)
|
||||
context.strokePath()
|
||||
|
||||
// Native PiP shows a left chevron on the peek strip when tucked right.
|
||||
let arrowOpacity = max(0, min(1, (tuckProgress - 0.55) / 0.35))
|
||||
if arrowOpacity > 0.01 {
|
||||
drawEdgeChevron(in: context, cardRect: rect, opacity: arrowOpacity)
|
||||
}
|
||||
|
||||
guard let logo = cachedLogo else { return }
|
||||
let logoOpacity = 1 - arrowOpacity
|
||||
guard logoOpacity > 0.01 else { return }
|
||||
|
||||
let maxLogoSide = min(rect.width, rect.height) * 0.52
|
||||
let logoAspect = CGFloat(logo.width) / CGFloat(max(logo.height, 1))
|
||||
let logoSize: CGSize
|
||||
if logoAspect >= 1 {
|
||||
logoSize = CGSize(width: maxLogoSide, height: maxLogoSide / logoAspect)
|
||||
} else {
|
||||
logoSize = CGSize(width: maxLogoSide * logoAspect, height: maxLogoSide)
|
||||
}
|
||||
let logoRect = CGRect(
|
||||
x: rect.midX - logoSize.width / 2,
|
||||
y: rect.midY - logoSize.height / 2,
|
||||
width: logoSize.width,
|
||||
height: logoSize.height
|
||||
)
|
||||
|
||||
// Unflip locally so the CGImage is not drawn upside-down.
|
||||
context.saveGState()
|
||||
context.setAlpha(logoOpacity)
|
||||
context.translateBy(x: logoRect.minX, y: logoRect.maxY)
|
||||
context.scaleBy(x: 1, y: -1)
|
||||
context.interpolationQuality = .high
|
||||
context.draw(logo, in: CGRect(origin: .zero, size: logoSize))
|
||||
context.restoreGState()
|
||||
}
|
||||
|
||||
/// Left-pointing chevron on the visible peek strip (like system PiP).
|
||||
private func drawEdgeChevron(
|
||||
in context: CGContext,
|
||||
cardRect: CGRect,
|
||||
opacity: CGFloat
|
||||
) {
|
||||
// Anchor in the leftmost ~28% of the card — that strip stays on-screen
|
||||
// when tucked to the right edge.
|
||||
let peekWidth = cardRect.width * 0.28
|
||||
let center = CGPoint(
|
||||
x: cardRect.minX + peekWidth * 0.5,
|
||||
y: cardRect.midY
|
||||
)
|
||||
let halfH: CGFloat = 11
|
||||
let halfW: CGFloat = 7
|
||||
|
||||
let path = UIBezierPath()
|
||||
path.move(to: CGPoint(x: center.x + halfW, y: center.y - halfH))
|
||||
path.addLine(to: CGPoint(x: center.x - halfW, y: center.y))
|
||||
path.addLine(to: CGPoint(x: center.x + halfW, y: center.y + halfH))
|
||||
|
||||
context.saveGState()
|
||||
context.setStrokeColor(UIColor.white.withAlphaComponent(opacity).cgColor)
|
||||
context.setLineWidth(3)
|
||||
context.setLineCap(.round)
|
||||
context.setLineJoin(.round)
|
||||
context.addPath(path.cgPath)
|
||||
context.strokePath()
|
||||
context.restoreGState()
|
||||
}
|
||||
|
||||
/// 0 = rest (visible), 1 = tucked at right edge.
|
||||
private func cardTravelProgress() -> CGFloat {
|
||||
let started = animationStartedAt ?? CACurrentMediaTime()
|
||||
if animationStartedAt == nil {
|
||||
animationStartedAt = started
|
||||
}
|
||||
let t = (CACurrentMediaTime() - started)
|
||||
.truncatingRemainder(dividingBy: Canvas.loopDuration)
|
||||
|
||||
// 0.0–0.6 rest → 0.6–2.0 slide out → 2.0–3.0 hold → 3.0–4.2 return
|
||||
if t < 0.6 {
|
||||
return 0
|
||||
}
|
||||
if t < 2.0 {
|
||||
return smoothstep((t - 0.6) / 1.4)
|
||||
}
|
||||
if t < 3.0 {
|
||||
return 1
|
||||
}
|
||||
return 1 - smoothstep((t - 3.0) / 1.2)
|
||||
}
|
||||
|
||||
private func smoothstep(_ x: CGFloat) -> CGFloat {
|
||||
let c = min(max(x, 0), 1)
|
||||
return c * c * (3 - 2 * c)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - AVPictureInPictureControllerDelegate
|
||||
|
||||
extension FlowPictureInPictureController: AVPictureInPictureControllerDelegate {
|
||||
extension FlowPictureInPictureController: @preconcurrency AVPictureInPictureControllerDelegate {
|
||||
func pictureInPictureControllerDidStartPictureInPicture(
|
||||
_ pictureInPictureController: AVPictureInPictureController
|
||||
) {
|
||||
isPictureInPictureActive = true
|
||||
lastSystemStartFailure = nil
|
||||
}
|
||||
|
||||
func pictureInPictureControllerDidStopPictureInPicture(
|
||||
@@ -258,6 +586,16 @@ extension FlowPictureInPictureController: AVPictureInPictureControllerDelegate {
|
||||
onUserDismissed?()
|
||||
}
|
||||
|
||||
func pictureInPictureController(
|
||||
_ pictureInPictureController: AVPictureInPictureController,
|
||||
failedToStartPictureInPictureWithError error: Error
|
||||
) {
|
||||
// First attempts often fail while the sample-buffer source is still
|
||||
// warming; keep retrying via the display link / auto-inline path.
|
||||
lastSystemStartFailure = error
|
||||
FlowDiagnostics.log("PiP start attempt failed (will retry): \(error.localizedDescription)")
|
||||
}
|
||||
|
||||
func pictureInPictureController(
|
||||
_ pictureInPictureController: AVPictureInPictureController,
|
||||
restoreUserInterfaceForPictureInPictureStopWithCompletionHandler completionHandler: @escaping (Bool) -> Void
|
||||
@@ -268,22 +606,29 @@ extension FlowPictureInPictureController: AVPictureInPictureControllerDelegate {
|
||||
|
||||
// MARK: - AVPictureInPictureSampleBufferPlaybackDelegate
|
||||
|
||||
extension FlowPictureInPictureController: AVPictureInPictureSampleBufferPlaybackDelegate {
|
||||
extension FlowPictureInPictureController: @preconcurrency AVPictureInPictureSampleBufferPlaybackDelegate {
|
||||
func pictureInPictureController(
|
||||
_ pictureInPictureController: AVPictureInPictureController,
|
||||
setPlaying playing: Bool
|
||||
) {
|
||||
if playing {
|
||||
if animationStartedAt == nil {
|
||||
animationStartedAt = CACurrentMediaTime()
|
||||
}
|
||||
startFramePump()
|
||||
} else {
|
||||
stopFramePump()
|
||||
// Do not stop the frame pump on pause — sample-buffer PiP keep-alive
|
||||
// must keep feeding frames so auto-inline can resume.
|
||||
animationStartedAt = CACurrentMediaTime()
|
||||
startFramePump()
|
||||
}
|
||||
}
|
||||
|
||||
func pictureInPictureControllerTimeRangeForPlayback(
|
||||
_ pictureInPictureController: AVPictureInPictureController
|
||||
) -> CMTimeRange {
|
||||
CMTimeRange(start: .zero, duration: CMTime(value: 3600, timescale: 1))
|
||||
// Live / unbounded content — finite durations make PiP stuck loading.
|
||||
CMTimeRange(start: .zero, duration: .positiveInfinity)
|
||||
}
|
||||
|
||||
func pictureInPictureControllerIsPlaybackPaused(
|
||||
|
||||
@@ -84,13 +84,33 @@ final class FlowSessionManager: ObservableObject {
|
||||
/// Initial proof window — cold mic sessions often need >2.5s after app switch.
|
||||
private static let coldStartAudioProofTimeout: TimeInterval = 6
|
||||
|
||||
private var keepAliveMode: FlowKeepAliveMode {
|
||||
FlowSessionPolicy.keepAliveMode()
|
||||
}
|
||||
|
||||
private var usesPiPKeepAlive: Bool {
|
||||
FlowSessionPolicy.keepAliveMode() == .pictureInPicture
|
||||
keepAliveMode == .pictureInPicture
|
||||
}
|
||||
|
||||
func attachPiPHostView(_ view: UIView) {
|
||||
pipController.attachHostView(view)
|
||||
}
|
||||
|
||||
/// Live Activity is mutually exclusive with PiP keep-alive.
|
||||
private func updateLiveActivityPhase(_ phase: FlowActivityAttributes.ContentState.Phase) {
|
||||
guard !usesPiPKeepAlive else { return }
|
||||
FlowLiveActivityController.update(phase: phase)
|
||||
}
|
||||
|
||||
private func startLiveActivityIfNeeded() {
|
||||
guard !usesPiPKeepAlive else {
|
||||
// Sweep any orphan island left from a previous Live Activity session.
|
||||
FlowLiveActivityController.clearOrphanedActivities()
|
||||
return
|
||||
}
|
||||
FlowLiveActivityController.startSession()
|
||||
}
|
||||
|
||||
/// Guards the once-per-process launch reconciliation (scene reconnects
|
||||
/// recreate the `@StateObject`-owned manager within the same process).
|
||||
private static var didRunLaunchReconciliation = false
|
||||
@@ -463,8 +483,14 @@ final class FlowSessionManager: ObservableObject {
|
||||
resumeAfterForeground()
|
||||
case .inactive:
|
||||
writeHeartbeatIfActive()
|
||||
if usesPiPKeepAlive, isActive {
|
||||
pipController.prepareForBackgroundAutoStart()
|
||||
}
|
||||
case .background:
|
||||
setAppForeground(false)
|
||||
if usesPiPKeepAlive, isActive {
|
||||
pipController.prepareForBackgroundAutoStart()
|
||||
}
|
||||
if coldStartContext != nil {
|
||||
dismissColdStartOverlay()
|
||||
}
|
||||
@@ -743,21 +769,22 @@ final class FlowSessionManager: ObservableObject {
|
||||
}
|
||||
|
||||
if usesPiPKeepAlive {
|
||||
let pipReady = await pipController.startAndWait()
|
||||
guard pipReady else {
|
||||
let message = AppL10n.string("flow.pip.error.unavailable")
|
||||
switch await pipController.startAndWait() {
|
||||
case .started:
|
||||
activateFlowSessionAfterPiPProof(duration: duration)
|
||||
traceState("startSessionAsync.ready")
|
||||
debug("Flow session started (PiP keep-alive), mic released between utterances")
|
||||
case .failed(let failure):
|
||||
let message = AppL10n.string(failure.localizationKey)
|
||||
sessionWarning = message
|
||||
traceState("startSessionAsync.failed", extra: "reason=pipUnavailable")
|
||||
traceState("startSessionAsync.failed", extra: "reason=pipUnavailable failure=\(failure)")
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
if isColdStartHandoff {
|
||||
showColdStartAudioFailure(message: message)
|
||||
showColdStartPipFailure(message: message)
|
||||
scheduleColdStartRecovery(duration: duration)
|
||||
}
|
||||
debug("PiP keep-alive failed to start")
|
||||
return
|
||||
debug("PiP keep-alive failed to start: \(failure)")
|
||||
}
|
||||
activateFlowSessionAfterPiPProof(duration: duration)
|
||||
traceState("startSessionAsync.ready")
|
||||
debug("Flow session started (PiP keep-alive), mic released between utterances")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -816,7 +843,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
bindSessionASRIfNeeded()
|
||||
scheduleASRWarmup()
|
||||
FlowLiveActivityController.startSession()
|
||||
startLiveActivityIfNeeded()
|
||||
|
||||
refreshHostReady()
|
||||
traceState("activateFlowSessionAfterPiPProof.done")
|
||||
@@ -841,7 +868,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
bindSessionASRIfNeeded()
|
||||
scheduleASRWarmup()
|
||||
FlowLiveActivityController.startSession()
|
||||
startLiveActivityIfNeeded()
|
||||
|
||||
refreshHostReady()
|
||||
traceState("activateFlowSessionAfterAudioProof.done")
|
||||
@@ -851,6 +878,20 @@ final class FlowSessionManager: ObservableObject {
|
||||
guard isColdStartHandoff, isActive else { return }
|
||||
if usesPiPKeepAlive {
|
||||
sessionWarning = nil
|
||||
if !pipController.isPictureInPictureActive {
|
||||
switch await pipController.startAndWait() {
|
||||
case .started:
|
||||
break
|
||||
case .failed(let failure):
|
||||
let message = AppL10n.string(failure.localizationKey)
|
||||
sessionWarning = message
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
showColdStartPipFailure(message: message)
|
||||
scheduleColdStartRecovery(duration: nil)
|
||||
debug("existing PiP session failed cold-start restart: \(failure)")
|
||||
return
|
||||
}
|
||||
}
|
||||
refreshHostReady()
|
||||
handleColdStartAfterSessionReady()
|
||||
return
|
||||
@@ -890,9 +931,16 @@ final class FlowSessionManager: ObservableObject {
|
||||
debug("cold-start handoff ignored: session busy with an utterance")
|
||||
return
|
||||
}
|
||||
let message = AppL10n.string("flow.coldStart.error.audioTimeout")
|
||||
sessionWarning = message
|
||||
showColdStartAudioFailure(message: message)
|
||||
let message: String
|
||||
if usesPiPKeepAlive {
|
||||
message = AppL10n.string("flow.pip.error.notPossible")
|
||||
sessionWarning = message
|
||||
showColdStartPipFailure(message: message)
|
||||
} else {
|
||||
message = AppL10n.string("flow.coldStart.error.audioTimeout")
|
||||
sessionWarning = message
|
||||
showColdStartAudioFailure(message: message)
|
||||
}
|
||||
scheduleColdStartRecovery(duration: nil)
|
||||
debug("cold-start blocked: host ready contract not published")
|
||||
return
|
||||
@@ -903,7 +951,11 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
private func presentColdStartReadyOverlay() {
|
||||
let hostEntry = HostReturnService.pendingHostEntry()
|
||||
coldStartContext = FlowColdStartContext(hostEntry: hostEntry, state: .ready)
|
||||
coldStartContext = FlowColdStartContext(
|
||||
hostEntry: hostEntry,
|
||||
state: .ready,
|
||||
keepAliveMode: keepAliveMode
|
||||
)
|
||||
scheduleAutoReturnToHostIfNeeded(hostEntry: hostEntry)
|
||||
}
|
||||
|
||||
@@ -929,17 +981,23 @@ final class FlowSessionManager: ObservableObject {
|
||||
coldStartRecoveryTask = Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
if self.usesPiPKeepAlive {
|
||||
let recovered = await self.pipController.startAndWait()
|
||||
self.traceState("coldStartRecovery.pip", extra: "recovered=\(recovered)")
|
||||
let outcome = await self.pipController.startAndWait()
|
||||
self.traceState("coldStartRecovery.pip", extra: "outcome=\(outcome)")
|
||||
guard !Task.isCancelled, self.isColdStartHandoff else { return }
|
||||
if recovered {
|
||||
switch outcome {
|
||||
case .started:
|
||||
if self.isActive {
|
||||
self.sessionWarning = nil
|
||||
self.refreshHostReady()
|
||||
self.handleColdStartAfterSessionReady()
|
||||
} else {
|
||||
self.activateFlowSessionAfterPiPProof(duration: duration)
|
||||
self.handleColdStartAfterSessionReady()
|
||||
}
|
||||
case .failed(let failure):
|
||||
let message = AppL10n.string(failure.localizationKey)
|
||||
self.sessionWarning = message
|
||||
self.showColdStartPipFailure(message: message)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -995,7 +1053,8 @@ final class FlowSessionManager: ObservableObject {
|
||||
private func showColdStartPreparing() {
|
||||
coldStartContext = FlowColdStartContext(
|
||||
hostEntry: HostReturnService.pendingHostEntry(),
|
||||
state: .preparing
|
||||
state: .preparing,
|
||||
keepAliveMode: keepAliveMode
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1003,7 +1062,8 @@ final class FlowSessionManager: ObservableObject {
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
coldStartContext = FlowColdStartContext(
|
||||
hostEntry: HostReturnService.pendingHostEntry(),
|
||||
state: .failed(.permission(message: permissionWarningMessage()))
|
||||
state: .failed(.permission(message: permissionWarningMessage())),
|
||||
keepAliveMode: keepAliveMode
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1011,7 +1071,17 @@ final class FlowSessionManager: ObservableObject {
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
coldStartContext = FlowColdStartContext(
|
||||
hostEntry: HostReturnService.pendingHostEntry(),
|
||||
state: .failed(.audio(message: message))
|
||||
state: .failed(.audio(message: message)),
|
||||
keepAliveMode: keepAliveMode
|
||||
)
|
||||
}
|
||||
|
||||
private func showColdStartPipFailure(message: String) {
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
coldStartContext = FlowColdStartContext(
|
||||
hostEntry: HostReturnService.pendingHostEntry(),
|
||||
state: .failed(.pip(message: message)),
|
||||
keepAliveMode: .pictureInPicture
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1221,6 +1291,8 @@ final class FlowSessionManager: ObservableObject {
|
||||
guard usesPiPKeepAlive, capture.running else { return }
|
||||
guard !isUtteranceRecording, !isUtteranceProcessing else { return }
|
||||
capture.stop()
|
||||
// Capture deactivates AVAudioSession; restore playback so PiP stays eligible.
|
||||
_ = pipController.reassertKeepAliveAudioSession()
|
||||
pipController.updateWaveformLevels([])
|
||||
refreshHostReady()
|
||||
}
|
||||
@@ -1274,7 +1346,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
utteranceRecordingStartedAt = Date()
|
||||
startUtteranceSafetyTimer()
|
||||
refreshHostReady()
|
||||
FlowLiveActivityController.update(phase: .recording)
|
||||
updateLiveActivityPhase(.recording)
|
||||
FlowDiagnostics.log(
|
||||
"beginUtterance engine=\(store.engineMode) " +
|
||||
"asrType=\(type(of: asr)) pipelined=true " +
|
||||
@@ -1349,7 +1421,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
utteranceSafetyTask?.cancel()
|
||||
utteranceSafetyTask = nil
|
||||
refreshHostReady()
|
||||
FlowLiveActivityController.update(phase: .processing)
|
||||
updateLiveActivityPhase(.processing)
|
||||
|
||||
// Snapshot pipelined partial before drain — fallback if the final chunk ASR drops tail text.
|
||||
bestPartialSnapshot = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
@@ -1402,7 +1474,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
chunkWarnings = []
|
||||
currentUtteranceId = nil
|
||||
currentCommandSeq = 0
|
||||
FlowLiveActivityController.update(phase: .idle)
|
||||
updateLiveActivityPhase(.idle)
|
||||
refreshHostReady()
|
||||
debug("utterance aborted")
|
||||
}
|
||||
@@ -1432,7 +1504,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
storeCurrentError(message, kind: kind)
|
||||
currentUtteranceId = nil
|
||||
currentCommandSeq = 0
|
||||
FlowLiveActivityController.update(phase: .idle)
|
||||
updateLiveActivityPhase(.idle)
|
||||
refreshHostReady()
|
||||
debug("utterance failed: \(message)")
|
||||
}
|
||||
@@ -1458,7 +1530,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
storeCurrentError(message, kind: kind)
|
||||
currentUtteranceId = nil
|
||||
currentCommandSeq = 0
|
||||
FlowLiveActivityController.update(phase: .idle)
|
||||
updateLiveActivityPhase(.idle)
|
||||
refreshHostReady()
|
||||
debug("utterance processing failed: \(message)")
|
||||
}
|
||||
@@ -1638,7 +1710,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
let wasProcessing = isUtteranceProcessing
|
||||
isUtteranceProcessing = false
|
||||
FlowLiveActivityController.update(phase: .idle)
|
||||
updateLiveActivityPhase(.idle)
|
||||
if isActive {
|
||||
touchSessionActivity()
|
||||
}
|
||||
@@ -1856,7 +1928,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
FlowSessionBridge.writeHeartbeat()
|
||||
self.refreshHostReady()
|
||||
tick += 1
|
||||
if tick % liveActivityKeepAliveEveryTicks == 0 {
|
||||
if !self.usesPiPKeepAlive, tick % liveActivityKeepAliveEveryTicks == 0 {
|
||||
FlowLiveActivityController.keepAlive()
|
||||
}
|
||||
try? await Task.sleep(nanoseconds: 1_000_000_000)
|
||||
|
||||
@@ -18,17 +18,18 @@ enum AppTab: Int, CaseIterable {
|
||||
var icon: MaterialIconName {
|
||||
switch self {
|
||||
case .keyboard: return .keyboard
|
||||
case .history: return .menuBook
|
||||
case .history: return .menuBook // unused — history uses SF Symbol
|
||||
case .dictionary: return .menuBook // unused — dictionary uses SF Symbol
|
||||
case .styles: return .menuBook // unused — styles uses SF Symbol
|
||||
case .settings: return .settings
|
||||
}
|
||||
}
|
||||
|
||||
/// Filled SF Symbol override for the dictionary tab.
|
||||
/// SF Symbol overrides shared with the Mac and iPad sidebars.
|
||||
var sfSymbol: String? {
|
||||
switch self {
|
||||
case .dictionary: return "square.stack.3d.down.right.fill"
|
||||
case .history: return "clock.arrow.circlepath"
|
||||
case .dictionary: return "character.book.closed"
|
||||
case .styles: return "text.badge.star"
|
||||
default: return nil
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ import OSGKeyboardShared
|
||||
struct FlowColdStartContext: Equatable {
|
||||
let hostEntry: HostAppEntry?
|
||||
var state: FlowColdStartState
|
||||
/// Drives preparing / PiP-specific copy (Live Activity vs picture-in-picture).
|
||||
var keepAliveMode: FlowKeepAliveMode
|
||||
}
|
||||
|
||||
enum FlowColdStartState: Equatable {
|
||||
@@ -24,6 +26,8 @@ enum FlowColdStartState: Equatable {
|
||||
enum FlowColdStartFailure: Equatable {
|
||||
case permission(message: String)
|
||||
case audio(message: String)
|
||||
/// Picture-in-picture keep-alive could not be proven active.
|
||||
case pip(message: String)
|
||||
}
|
||||
|
||||
struct FlowColdStartOverlay: View {
|
||||
@@ -119,7 +123,7 @@ struct FlowColdStartOverlay: View {
|
||||
ProgressView()
|
||||
.tint(palette.accent)
|
||||
.scaleEffect(1.1)
|
||||
.accessibilityLabel(AppL10n.string("flow.coldStart.preparing"))
|
||||
.accessibilityLabel(preparingTitle)
|
||||
case .ready:
|
||||
Image(systemName: "checkmark.circle.fill")
|
||||
.font(.system(size: 26, weight: .semibold))
|
||||
@@ -142,7 +146,7 @@ struct FlowColdStartOverlay: View {
|
||||
switch failure {
|
||||
case .permission:
|
||||
linkButton(AppL10n.string("flow.coldStart.action.settings"), action: onOpenSettings)
|
||||
case .audio:
|
||||
case .audio, .pip:
|
||||
linkButton(AppL10n.string("flow.coldStart.action.retry"), action: onRetry)
|
||||
}
|
||||
}
|
||||
@@ -157,10 +161,19 @@ struct FlowColdStartOverlay: View {
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
|
||||
private var preparingTitle: String {
|
||||
switch context.keepAliveMode {
|
||||
case .pictureInPicture:
|
||||
return AppL10n.string("flow.coldStart.preparing.pip")
|
||||
case .liveActivity:
|
||||
return AppL10n.string("flow.coldStart.preparing")
|
||||
}
|
||||
}
|
||||
|
||||
private var title: String {
|
||||
switch context.state {
|
||||
case .preparing:
|
||||
return AppL10n.string("flow.coldStart.preparing")
|
||||
return preparingTitle
|
||||
case .ready:
|
||||
return AppL10n.string("flow.coldStart.title")
|
||||
case .failed(let failure):
|
||||
@@ -169,6 +182,8 @@ struct FlowColdStartOverlay: View {
|
||||
return AppL10n.string("flow.coldStart.permission.title")
|
||||
case .audio:
|
||||
return AppL10n.string("flow.coldStart.audio.title")
|
||||
case .pip:
|
||||
return AppL10n.string("flow.coldStart.pip.title")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -176,14 +191,17 @@ struct FlowColdStartOverlay: View {
|
||||
private var message: String {
|
||||
switch context.state {
|
||||
case .preparing:
|
||||
return AppL10n.string("flow.coldStart.preparingHint")
|
||||
switch context.keepAliveMode {
|
||||
case .pictureInPicture:
|
||||
return AppL10n.string("flow.coldStart.preparingHint.pip")
|
||||
case .liveActivity:
|
||||
return AppL10n.string("flow.coldStart.preparingHint")
|
||||
}
|
||||
case .ready:
|
||||
return AppL10n.string("flow.coldStart.swipeHint")
|
||||
case .failed(let failure):
|
||||
switch failure {
|
||||
case .permission(let message):
|
||||
return message
|
||||
case .audio(let message):
|
||||
case .permission(let message), .audio(let message), .pip(let message):
|
||||
return message
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,15 +9,37 @@ import UIKit
|
||||
struct FlowPiPHostView: UIViewRepresentable {
|
||||
let attach: (UIView) -> Void
|
||||
|
||||
func makeUIView(context: Context) -> UIView {
|
||||
let view = UIView(frame: CGRect(x: 0, y: 0, width: 2, height: 2))
|
||||
func makeUIView(context: Context) -> FlowPiPHostUIView {
|
||||
// Non-trivial size: a 1×1 / fully invisible host often keeps
|
||||
// `isPictureInPicturePossible` false for sample-buffer sources.
|
||||
let view = FlowPiPHostUIView(frame: CGRect(x: 0, y: 0, width: 64, height: 36))
|
||||
view.isUserInteractionEnabled = false
|
||||
view.backgroundColor = .clear
|
||||
view.isOpaque = false
|
||||
view.onMovedToWindow = { [weak view] in
|
||||
guard let view else { return }
|
||||
attach(view)
|
||||
}
|
||||
attach(view)
|
||||
return view
|
||||
}
|
||||
|
||||
func updateUIView(_ uiView: UIView, context: Context) {
|
||||
func updateUIView(_ uiView: FlowPiPHostUIView, context: Context) {
|
||||
attach(uiView)
|
||||
}
|
||||
}
|
||||
|
||||
/// Reports window membership so PiP start can wait for a real hierarchy.
|
||||
final class FlowPiPHostUIView: UIView {
|
||||
var onMovedToWindow: (() -> Void)?
|
||||
|
||||
override func didMoveToWindow() {
|
||||
super.didMoveToWindow()
|
||||
onMovedToWindow?()
|
||||
}
|
||||
|
||||
override func layoutSubviews() {
|
||||
super.layoutSubviews()
|
||||
onMovedToWindow?()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,9 +92,13 @@ struct HomeView: View {
|
||||
let logoTopPadding = isCompact ? Spacing.lg : Spacing.xxl
|
||||
let logoBottomPadding = isCompact ? Spacing.lg : Spacing.xxl
|
||||
let extrasBottomPadding = isCompact ? Spacing.sm : Spacing.lg
|
||||
let statusTopPadding = isCompact ? Spacing.sm : Spacing.xl
|
||||
// 输入框最小高度:小屏可压得更矮,让底部状态行始终留在 tab 栏之上。
|
||||
let previewMinHeight: CGFloat = isCompact ? 72 : 160
|
||||
// 有警告/引导时进一步压低输入框下限,把垂直空间让给底部状态行。
|
||||
let previewMinHeight: CGFloat = {
|
||||
if showsFlowSessionExtras {
|
||||
return isCompact ? 44 : 88
|
||||
}
|
||||
return isCompact ? 72 : 160
|
||||
}()
|
||||
|
||||
ZStack(alignment: .top) {
|
||||
sessionHeaderGradient(height: gradientHeight)
|
||||
@@ -116,22 +120,17 @@ struct HomeView: View {
|
||||
.padding(.horizontal, Spacing.lg)
|
||||
.padding(.bottom, Spacing.md)
|
||||
|
||||
// 唯一的弹性区块:吸收全部剩余空间(大屏铺满、小屏优先让位)。
|
||||
// 弹性输入框:吸收剩余高度;底部状态通过 safeAreaInset 锚定在
|
||||
// tab 栏之上,警告变高时输入框自动变矮,不再被 dock 挡住。
|
||||
previewField(minHeight: previewMinHeight)
|
||||
.padding(.horizontal, Spacing.lg)
|
||||
.frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top)
|
||||
.layoutPriority(-1)
|
||||
|
||||
HStack(spacing: Spacing.sm) {
|
||||
engineStatusLine
|
||||
flowStatusFooter
|
||||
}
|
||||
.frame(maxWidth: .infinity, alignment: .center)
|
||||
.padding(.horizontal, Spacing.lg)
|
||||
.padding(.top, statusTopPadding)
|
||||
.padding(.bottom, Spacing.sm)
|
||||
}
|
||||
.frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top)
|
||||
.safeAreaInset(edge: .bottom, spacing: 0) {
|
||||
phoneStatusFooter
|
||||
}
|
||||
}
|
||||
.background(palette.background)
|
||||
.contentShape(Rectangle())
|
||||
@@ -143,6 +142,19 @@ struct HomeView: View {
|
||||
}
|
||||
}
|
||||
|
||||
/// Engine + Flow 状态行:作为 bottom inset,始终压在自定义 tab 栏之上。
|
||||
private var phoneStatusFooter: some View {
|
||||
HStack(spacing: Spacing.sm) {
|
||||
engineStatusLine
|
||||
flowStatusFooter
|
||||
}
|
||||
.frame(maxWidth: .infinity, alignment: .center)
|
||||
.padding(.horizontal, Spacing.lg)
|
||||
.padding(.top, Spacing.sm)
|
||||
.padding(.bottom, Spacing.sm)
|
||||
.background(palette.background.opacity(0.96))
|
||||
}
|
||||
|
||||
// MARK: - Wide layout (iPad / regular width)
|
||||
|
||||
private var wideBody: some View {
|
||||
@@ -486,7 +498,9 @@ struct HomeView: View {
|
||||
EngineServiceLabel.summary(
|
||||
engineMode: config.engineMode,
|
||||
providerId: config.providerId,
|
||||
model: config.model
|
||||
model: config.model,
|
||||
asrProviderId: config.asrProviderId,
|
||||
asrModel: config.asrModel
|
||||
)
|
||||
)
|
||||
.font(TypeStyle.caption2)
|
||||
|
||||
@@ -143,7 +143,9 @@ struct KeyboardPreviewSheet: View {
|
||||
EngineServiceLabel.summary(
|
||||
engineMode: config.engineMode,
|
||||
providerId: config.providerId,
|
||||
model: config.model
|
||||
model: config.model,
|
||||
asrProviderId: config.asrProviderId,
|
||||
asrModel: config.asrModel
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -11,15 +11,19 @@ import OSGKeyboardShared
|
||||
struct MainAppRoot: View {
|
||||
@Environment(\.scenePhase) private var scenePhase
|
||||
|
||||
@StateObject private var config = ProviderConfig.shared
|
||||
// Singleton is owned by `ProviderConfig.shared`, not by this view —
|
||||
// `@ObservedObject` keeps subscriptions correct across Settings replay.
|
||||
@ObservedObject private var config = ProviderConfig.shared
|
||||
@StateObject private var flowManager = FlowSessionManager()
|
||||
|
||||
var body: some View {
|
||||
Group {
|
||||
if config.hasCompletedOnboarding {
|
||||
MainTabView()
|
||||
.id("main")
|
||||
} else {
|
||||
OnboardingView(config: config)
|
||||
.id("onboarding")
|
||||
}
|
||||
}
|
||||
.environment(\.locale, config.uiLanguage.swiftUILocale)
|
||||
@@ -41,9 +45,11 @@ struct MainAppRoot: View {
|
||||
FlowPiPHostView { view in
|
||||
flowManager.attachPiPHostView(view)
|
||||
}
|
||||
.frame(width: 2, height: 2)
|
||||
.opacity(0.001)
|
||||
// Keep a small but real layer in the window hierarchy for PiP.
|
||||
.frame(width: 64, height: 36)
|
||||
.opacity(0.02)
|
||||
.allowsHitTesting(false)
|
||||
.accessibilityHidden(true)
|
||||
}
|
||||
.onAppear {
|
||||
flowManager.setAppForeground(scenePhase == .active)
|
||||
|
||||
@@ -39,7 +39,9 @@ struct MainTabView: View {
|
||||
.environment(\.isTabBarVisible, !isTabBarHidden)
|
||||
.safeAreaInset(edge: .bottom, spacing: 0) {
|
||||
if !isTabBarHidden {
|
||||
Color.clear.frame(height: 88)
|
||||
// Match floating dock + home-indicator clearance so
|
||||
// page footers / scroll ends sit above MinimalTabBar.
|
||||
Color.clear.frame(height: 100)
|
||||
}
|
||||
}
|
||||
.onPreferenceChange(TabBarHiddenPreferenceKey.self) { hidden in
|
||||
|
||||
@@ -176,13 +176,19 @@ struct OnboardingView: View {
|
||||
|
||||
private func advancePage() {
|
||||
refreshPermissionStatuses()
|
||||
withAnimation(Motion.soft) {
|
||||
if isLastPage {
|
||||
// Routing out of onboarding must NOT run inside an animation
|
||||
// transaction. Animating OnboardingView → MainTabView (plus a nested
|
||||
// onboardingPage reset and Flow activateOnForeground) can leave the
|
||||
// last page frozen even when hasCompletedOnboarding is already true.
|
||||
if isLastPage || nextVisiblePage(after: config.onboardingPage) == nil {
|
||||
var transaction = Transaction()
|
||||
transaction.disablesAnimations = true
|
||||
withTransaction(transaction) {
|
||||
config.hasCompletedOnboarding = true
|
||||
} else if let next = nextVisiblePage(after: config.onboardingPage) {
|
||||
}
|
||||
} else if let next = nextVisiblePage(after: config.onboardingPage) {
|
||||
withAnimation(Motion.soft) {
|
||||
config.onboardingPage = next
|
||||
} else {
|
||||
config.hasCompletedOnboarding = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -401,8 +401,12 @@ struct SettingsView: View {
|
||||
sectionHeader("settings.about.title")
|
||||
VStack(spacing: 0) {
|
||||
Button {
|
||||
config.hasCompletedOnboarding = false
|
||||
config.onboardingPage = 0
|
||||
var transaction = Transaction()
|
||||
transaction.disablesAnimations = true
|
||||
withTransaction(transaction) {
|
||||
config.hasCompletedOnboarding = false
|
||||
config.onboardingPage = 0
|
||||
}
|
||||
} label: {
|
||||
HStack(spacing: Spacing.sm) {
|
||||
Text("settings.onboarding.replay")
|
||||
@@ -582,7 +586,7 @@ private struct FlowKeepAliveModePickerRow: View {
|
||||
selection: Binding(
|
||||
get: { selection.rawValue },
|
||||
set: { newValue in
|
||||
selection = FlowKeepAliveMode(rawValue: newValue) ?? .liveActivity
|
||||
selection = FlowKeepAliveMode(rawValue: newValue) ?? .default
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
@@ -473,11 +473,19 @@
|
||||
/* Cold-start handoff (scheme B) */
|
||||
"flow.coldStart.title" = "Voice is ready";
|
||||
"flow.coldStart.preparing" = "Getting voice ready";
|
||||
"flow.coldStart.preparing.pip" = "Starting Picture in Picture";
|
||||
"flow.coldStart.preparingHint" = "Keep OSGKeyboard open for a moment while we start the microphone session.";
|
||||
"flow.coldStart.preparingHint.pip" = "Keep OSGKeyboard open while Picture in Picture starts. Then return to the keyboard to speak.";
|
||||
"flow.coldStart.permission.title" = "Permission required";
|
||||
"flow.coldStart.audio.title" = "Voice could not start";
|
||||
"flow.coldStart.pip.title" = "Picture in Picture could not start";
|
||||
"flow.coldStart.error.audioTimeout" = "The microphone did not become ready in time. It may be busy in another app. Please try again.";
|
||||
"flow.pip.error.unavailable" = "Picture in Picture could not start. Check that PiP is allowed for OSGKeyboard in Settings.";
|
||||
"flow.pip.error.unavailable" = "Picture in Picture could not start. Stay in the app and try again.";
|
||||
"flow.pip.error.unsupported" = "This device does not support Picture in Picture.";
|
||||
"flow.pip.error.hostNotReady" = "The Picture in Picture surface is not ready yet. Stay in the app and try again.";
|
||||
"flow.pip.error.notPossible" = "The system cannot start Picture in Picture right now. Keep the app in the foreground and try again.";
|
||||
"flow.pip.error.systemRejected" = "Picture in Picture was rejected by the system. Please try again shortly.";
|
||||
"flow.pip.error.timedOut" = "Picture in Picture did not appear in time. Stay in the app and try again.";
|
||||
"flow.coldStart.action.settings" = "Open Settings";
|
||||
"flow.coldStart.action.retry" = "Try Again";
|
||||
"flow.coldStart.swipeHint" = "Swipe right along the bottom bar\nto return to your previous app.";
|
||||
|
||||
@@ -472,11 +472,19 @@
|
||||
/* 冷启动兜底(方案 B) */
|
||||
"flow.coldStart.title" = "语音已就绪";
|
||||
"flow.coldStart.preparing" = "正在就绪";
|
||||
"flow.coldStart.preparing.pip" = "正在启动画中画";
|
||||
"flow.coldStart.preparingHint" = "请先停留片刻,我们正在启动麦克风会话。";
|
||||
"flow.coldStart.preparingHint.pip" = "请先停留片刻,我们正在启动画中画保活。就绪后可返回键盘直接说话。";
|
||||
"flow.coldStart.permission.title" = "需要权限";
|
||||
"flow.coldStart.audio.title" = "语音暂时无法启动";
|
||||
"flow.coldStart.pip.title" = "画中画暂时无法启动";
|
||||
"flow.coldStart.error.audioTimeout" = "麦克风未能及时就绪,可能正被通话或其他 App 占用。请稍后重试。";
|
||||
"flow.pip.error.unavailable" = "无法启动画中画,请在系统设置中允许 OSGKeyboard 使用画中画。";
|
||||
"flow.pip.error.unavailable" = "无法启动画中画,请留在 App 内重试。";
|
||||
"flow.pip.error.unsupported" = "此设备不支持画中画。";
|
||||
"flow.pip.error.hostNotReady" = "画中画界面尚未就绪,请留在 App 内稍后重试。";
|
||||
"flow.pip.error.notPossible" = "系统暂时无法开启画中画,请保持 App 在前台后重试。";
|
||||
"flow.pip.error.systemRejected" = "画中画启动被系统拒绝,请稍后重试。";
|
||||
"flow.pip.error.timedOut" = "画中画未能及时出现,请留在 App 内重试。";
|
||||
"flow.coldStart.action.settings" = "前往设置";
|
||||
"flow.coldStart.action.retry" = "重试";
|
||||
"flow.coldStart.swipeHint" = "沿屏幕底部的横条,从左向右滑动\n返回上一个 App。";
|
||||
|
||||
Reference in New Issue
Block a user