feat(app): add resilient Flow recovery and privacy-safe analytics
Unify PiP recovery across startup and foreground transitions, surface actionable status, and add privacy-safe analytics plus the refreshed onboarding and account experience. Update documentation assets and advance the release build to 84.
This commit is contained in:
@@ -31,6 +31,67 @@ enum FlowUtteranceLifecyclePolicy {
|
||||
}
|
||||
}
|
||||
|
||||
enum FlowPiPActivationTrigger: String, Sendable {
|
||||
case coldStart
|
||||
case foreground
|
||||
case existingSession
|
||||
case startupRecovery
|
||||
case unexpectedStop
|
||||
case healthCheck
|
||||
case manualRetry
|
||||
}
|
||||
|
||||
enum FlowPiPReconciliationDecision: Equatable, Sendable {
|
||||
case noSessionIntent
|
||||
case alreadyActive
|
||||
case waitForForeground
|
||||
case startRecovery
|
||||
}
|
||||
|
||||
enum FlowPiPRecoveryPolicy {
|
||||
static let maxAttempts = 3
|
||||
static let totalBudget: TimeInterval = 5
|
||||
|
||||
static func reconciliationDecision(
|
||||
wantsActiveSession: Bool,
|
||||
isAppForeground: Bool,
|
||||
isPictureInPictureActive: Bool
|
||||
) -> FlowPiPReconciliationDecision {
|
||||
guard wantsActiveSession else { return .noSessionIntent }
|
||||
if isPictureInPictureActive { return .alreadyActive }
|
||||
return isAppForeground ? .startRecovery : .waitForForeground
|
||||
}
|
||||
|
||||
static func canContinue(
|
||||
operation: UInt64,
|
||||
currentOperation: UInt64,
|
||||
wantsActiveSession: Bool,
|
||||
isAppForeground: Bool,
|
||||
taskIsCancelled: Bool
|
||||
) -> Bool {
|
||||
operation == currentOperation
|
||||
&& wantsActiveSession
|
||||
&& isAppForeground
|
||||
&& !taskIsCancelled
|
||||
}
|
||||
|
||||
static func retryDelay(beforeAttempt attempt: Int) -> TimeInterval {
|
||||
switch attempt {
|
||||
case 1: return 0
|
||||
case 2: return 0.25
|
||||
default: return 0.6
|
||||
}
|
||||
}
|
||||
|
||||
static func hostTimeout(attempt: Int, remainingBudget: TimeInterval) -> TimeInterval {
|
||||
min(attempt == 1 ? 1.5 : 0.25, max(0, remainingBudget))
|
||||
}
|
||||
|
||||
static func activeTimeout(remainingBudget: TimeInterval) -> TimeInterval {
|
||||
min(1.4, max(0, remainingBudget))
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
final class FlowSessionManager: ObservableObject {
|
||||
private struct PrefilledAIRequest {
|
||||
@@ -45,12 +106,14 @@ final class FlowSessionManager: ObservableObject {
|
||||
@Published private(set) var sessionExpiresAt: Date?
|
||||
/// Non-nil when continuous capture failed or permissions are missing.
|
||||
@Published private(set) var sessionWarning: String?
|
||||
@Published private(set) var pipLifecycleState: FlowPiPLifecycleState = .inactive
|
||||
|
||||
private let capture = FlowContinuousCapture()
|
||||
private let pipController = FlowPictureInPictureController()
|
||||
private let pipController: any FlowPictureInPictureControlling
|
||||
private let store = AppGroupStore()
|
||||
private let analyticsClient: any AnalyticsClient
|
||||
/// Cloud / local polish via the user-configured LLM provider.
|
||||
private let polisher = PolishingService()
|
||||
private let polisher: PolishingService
|
||||
/// AI-mode turns are intentionally process-local and never persisted.
|
||||
private let aiConversations = AIConversationStore()
|
||||
/// Cached ASR instance. The only on-device backend is iOS
|
||||
@@ -98,6 +161,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
private var pendingSourceHistoryEntryID: UUID?
|
||||
private var pendingSourceHistoryEntryRevision: Int64?
|
||||
private var pendingAIConversationID: UUID?
|
||||
private var pendingManagedRequestPurpose: ManagedGatewayRequestPurpose?
|
||||
private var pendingProcessingDeadlineAt: TimeInterval?
|
||||
private var pendingStopUtteranceId: UUID?
|
||||
private var currentCommandSeq: Int64 = 0
|
||||
@@ -110,6 +174,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
@Published private(set) var isUtteranceProcessing = false
|
||||
private var finalizeTask: Task<Void, Never>?
|
||||
private var asrTask: Task<Void, Never>?
|
||||
private var transcriptionAnalyticsOperation: (any AnalyticsAIOperation)?
|
||||
private var utteranceSafetyTask: Task<Void, Never>?
|
||||
private var chunkedPipeline: ChunkedUtterancePipeline?
|
||||
private var currentPartial = ""
|
||||
@@ -131,7 +196,10 @@ final class FlowSessionManager: ObservableObject {
|
||||
private var isAppForeground = false
|
||||
/// True while handling a keyboard-initiated `startflow` cold start.
|
||||
private var isColdStartHandoff = false
|
||||
private var coldStartRecoveryTask: Task<Void, Never>?
|
||||
/// User intent outlives a transient PiP loss; only explicit teardown clears it.
|
||||
private var wantsActiveSession = false
|
||||
/// Invalidates late async completions after retry, stop, or process teardown.
|
||||
private var pipRecoveryOperation: UInt64 = 0
|
||||
var shouldDeferHostHeavyWork: Bool {
|
||||
isStarting
|
||||
|| startingUtteranceId != nil
|
||||
@@ -148,7 +216,13 @@ final class FlowSessionManager: ObservableObject {
|
||||
/// recreate the `@StateObject`-owned manager within the same process).
|
||||
private static var didRunLaunchReconciliation = false
|
||||
|
||||
init() {
|
||||
init(
|
||||
analyticsClient: any AnalyticsClient = NoopAnalyticsClient(),
|
||||
pipController: any FlowPictureInPictureControlling = FlowPictureInPictureController()
|
||||
) {
|
||||
self.analyticsClient = analyticsClient
|
||||
self.polisher = PolishingService(analyticsClient: analyticsClient)
|
||||
self.pipController = pipController
|
||||
// Sessions are (re)started explicitly on app foreground via
|
||||
// `activateOnForeground()`. We deliberately do NOT silently reattach a
|
||||
// stored session here — after a force-quit that would resurrect capture
|
||||
@@ -194,10 +268,9 @@ final class FlowSessionManager: ObservableObject {
|
||||
kind: .recognitionInterrupted
|
||||
)
|
||||
}
|
||||
pipController.onUserDismissed = { [weak self] in
|
||||
guard let self, self.isActive else { return }
|
||||
self.debug("PiP dismissed by user — ending Flow session")
|
||||
self.endSession()
|
||||
pipController.onUnexpectedStop = { [weak self] in
|
||||
guard let self, self.wantsActiveSession else { return }
|
||||
self.handleUnexpectedPiPStop()
|
||||
}
|
||||
FlowTerminationCoordinator.register(self)
|
||||
}
|
||||
@@ -257,25 +330,25 @@ final class FlowSessionManager: ObservableObject {
|
||||
}
|
||||
|
||||
if isActive {
|
||||
wantsActiveSession = true
|
||||
extendSession(duration: duration)
|
||||
refreshHostReady()
|
||||
if pipController.isPictureInPictureActive {
|
||||
pipLifecycleState = .active
|
||||
refreshHostReady()
|
||||
} else {
|
||||
ensurePiPActive(
|
||||
trigger: coldStart ? .coldStart : .existingSession,
|
||||
duration: duration
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
guard !isStarting else {
|
||||
traceState("startSession.ignored", extra: "reason=alreadyStarting")
|
||||
return
|
||||
}
|
||||
// Claim the flag synchronously: on a cold start the URL router and
|
||||
// `activateOnForeground()` both fire in the same runloop turn, and
|
||||
// setting it inside the async body let two start bodies interleave.
|
||||
isStarting = true
|
||||
|
||||
startTask?.cancel()
|
||||
startTask = Task { @MainActor [weak self] in
|
||||
await self?.startSessionAsync(duration: duration)
|
||||
self?.handleColdStartAfterSessionReady()
|
||||
}
|
||||
wantsActiveSession = true
|
||||
ensurePiPActive(
|
||||
trigger: coldStart ? .coldStart : .foreground,
|
||||
duration: duration
|
||||
)
|
||||
}
|
||||
|
||||
/// Clears App Group Flow state left behind when the host process was killed
|
||||
@@ -339,9 +412,19 @@ final class FlowSessionManager: ObservableObject {
|
||||
)
|
||||
}
|
||||
|
||||
/// User-requested recovery always discards the rejected AVKit generation.
|
||||
func retryPiPRecovery() {
|
||||
guard AppGroup.isAvailable else { return }
|
||||
guard AppPermissions.flowRequirementsMet else {
|
||||
sessionWarning = permissionWarningMessage()
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
return
|
||||
}
|
||||
wantsActiveSession = true
|
||||
ensurePiPActive(trigger: .manualRetry, duration: nil, forceReset: true)
|
||||
}
|
||||
|
||||
private func completeColdStartHandoff() {
|
||||
coldStartRecoveryTask?.cancel()
|
||||
coldStartRecoveryTask = nil
|
||||
isColdStartHandoff = false
|
||||
if isActive {
|
||||
refreshHostReady()
|
||||
@@ -360,9 +443,10 @@ final class FlowSessionManager: ObservableObject {
|
||||
)
|
||||
}
|
||||
|
||||
wantsActiveSession = false
|
||||
pipRecoveryOperation &+= 1
|
||||
pipLifecycleState = .inactive
|
||||
isColdStartHandoff = false
|
||||
coldStartRecoveryTask?.cancel()
|
||||
coldStartRecoveryTask = nil
|
||||
startTask?.cancel()
|
||||
startTask = nil
|
||||
startupAudioHealthTask?.cancel()
|
||||
@@ -433,7 +517,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
}
|
||||
|
||||
func endSession() {
|
||||
guard isActive else { return }
|
||||
guard isActive || isStarting || wantsActiveSession else { return }
|
||||
debug("Flow session ended")
|
||||
if isUtteranceRecording || isUtteranceProcessing,
|
||||
claimTerminal(utteranceId: currentUtteranceId) {
|
||||
@@ -444,9 +528,10 @@ final class FlowSessionManager: ObservableObject {
|
||||
)
|
||||
}
|
||||
|
||||
wantsActiveSession = false
|
||||
pipRecoveryOperation &+= 1
|
||||
pipLifecycleState = .inactive
|
||||
isColdStartHandoff = false
|
||||
coldStartRecoveryTask?.cancel()
|
||||
coldStartRecoveryTask = nil
|
||||
startTask?.cancel()
|
||||
startTask = nil
|
||||
startupAudioHealthTask?.cancel()
|
||||
@@ -493,6 +578,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
FlowSessionBridge.markSessionInactive()
|
||||
FlowSessionDarwin.postSessionChanged()
|
||||
isActive = false
|
||||
isStarting = false
|
||||
sessionExpiresAt = nil
|
||||
sessionWarning = nil
|
||||
currentPartial = ""
|
||||
@@ -516,19 +602,27 @@ final class FlowSessionManager: ObservableObject {
|
||||
case .active:
|
||||
setAppForeground(true)
|
||||
resumeAfterForeground()
|
||||
if wantsActiveSession, !pipController.isPictureInPictureActive {
|
||||
ensurePiPActive(trigger: .foreground, duration: nil)
|
||||
}
|
||||
case .inactive:
|
||||
setAppForeground(false)
|
||||
writeHeartbeatIfActive()
|
||||
if isActive {
|
||||
if isActive, pipController.isPictureInPictureActive {
|
||||
Task { @MainActor [weak self] in
|
||||
await self?.pipController.prepareForBackgroundAutoStart()
|
||||
}
|
||||
} else if wantsActiveSession {
|
||||
pausePiPRecoveryUntilForeground()
|
||||
}
|
||||
case .background:
|
||||
setAppForeground(false)
|
||||
if isActive {
|
||||
if isActive, pipController.isPictureInPictureActive {
|
||||
Task { @MainActor [weak self] in
|
||||
await self?.pipController.prepareForBackgroundAutoStart()
|
||||
}
|
||||
} else if wantsActiveSession {
|
||||
pausePiPRecoveryUntilForeground()
|
||||
}
|
||||
if isColdStartHandoff {
|
||||
completeColdStartHandoff()
|
||||
@@ -549,6 +643,33 @@ final class FlowSessionManager: ObservableObject {
|
||||
FlowSessionBridge.writeHeartbeat()
|
||||
}
|
||||
|
||||
private func handleUnexpectedPiPStop() {
|
||||
guard wantsActiveSession else { return }
|
||||
traceState("pip.unexpectedStop")
|
||||
pipLifecycleState = .waitingForForeground
|
||||
sessionWarning = nil
|
||||
if isUtteranceRecording {
|
||||
failUtterance(
|
||||
message: AppL10n.string("flow.error.recognitionInterrupted"),
|
||||
kind: .recognitionInterrupted
|
||||
)
|
||||
}
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
refreshHostReady()
|
||||
// Do not immediately fight another app for the system-owned PiP slot.
|
||||
// The scene-active path or the existing heartbeat performs reconciliation.
|
||||
}
|
||||
|
||||
private func pausePiPRecoveryUntilForeground() {
|
||||
startTask?.cancel()
|
||||
startTask = nil
|
||||
pipRecoveryOperation &+= 1
|
||||
pipController.resetGeneration()
|
||||
isStarting = false
|
||||
pipLifecycleState = .waitingForForeground
|
||||
refreshHostReady()
|
||||
}
|
||||
|
||||
private func resumeAfterForeground() {
|
||||
guard isActive else {
|
||||
return
|
||||
@@ -723,36 +844,192 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
// MARK: - Session start
|
||||
|
||||
private func startSessionAsync(duration: TimeInterval?) async {
|
||||
// `isStarting` was claimed synchronously in `startSession()`.
|
||||
defer { isStarting = false }
|
||||
private func ensurePiPActive(
|
||||
trigger: FlowPiPActivationTrigger,
|
||||
duration: TimeInterval?,
|
||||
forceReset: Bool = false
|
||||
) {
|
||||
switch FlowPiPRecoveryPolicy.reconciliationDecision(
|
||||
wantsActiveSession: wantsActiveSession,
|
||||
isAppForeground: isAppForeground,
|
||||
isPictureInPictureActive: pipController.isPictureInPictureActive
|
||||
) {
|
||||
case .noSessionIntent:
|
||||
return
|
||||
case .alreadyActive:
|
||||
pipLifecycleState = .active
|
||||
sessionWarning = nil
|
||||
if isActive {
|
||||
refreshHostReady()
|
||||
} else {
|
||||
activateFlowSessionAfterPiPProof(duration: duration)
|
||||
}
|
||||
handleColdStartAfterSessionReady()
|
||||
return
|
||||
case .waitForForeground:
|
||||
pipLifecycleState = .waitingForForeground
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
refreshHostReady()
|
||||
return
|
||||
case .startRecovery:
|
||||
break
|
||||
}
|
||||
|
||||
guard startTask == nil else {
|
||||
traceState(
|
||||
"pipRecovery.coalesced",
|
||||
extra: "trigger=\(trigger.rawValue)"
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
// Claim the flag synchronously: on a cold start the URL router and
|
||||
// `activateOnForeground()` both fire in the same runloop turn, and
|
||||
// setting it inside the async body let two start bodies interleave.
|
||||
isStarting = true
|
||||
pipRecoveryOperation &+= 1
|
||||
let operation = pipRecoveryOperation
|
||||
let recoveringExistingSession = isActive
|
||||
let resetFirstGeneration = forceReset
|
||||
|| recoveringExistingSession
|
||||
|| pipController.generation > 0
|
||||
pipLifecycleState = recoveringExistingSession
|
||||
? .recovering(attempt: 1, total: FlowPiPRecoveryPolicy.maxAttempts)
|
||||
: .preparing(attempt: 1, total: FlowPiPRecoveryPolicy.maxAttempts)
|
||||
|
||||
startTask = Task { @MainActor [weak self] in
|
||||
await self?.startSessionAsync(
|
||||
duration: duration,
|
||||
trigger: trigger,
|
||||
operation: operation,
|
||||
resetFirstGeneration: resetFirstGeneration
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private func startSessionAsync(
|
||||
duration: TimeInterval?,
|
||||
trigger: FlowPiPActivationTrigger,
|
||||
operation: UInt64,
|
||||
resetFirstGeneration: Bool
|
||||
) async {
|
||||
// `isStarting` was claimed synchronously in `ensurePiPActive()`.
|
||||
defer {
|
||||
if operation == pipRecoveryOperation {
|
||||
isStarting = false
|
||||
startTask = nil
|
||||
}
|
||||
}
|
||||
guard !Task.isCancelled else { return }
|
||||
traceState("startSessionAsync.begin")
|
||||
traceState("startSessionAsync.begin", extra: "trigger=\(trigger.rawValue)")
|
||||
sessionWarning = nil
|
||||
|
||||
guard AppPermissions.flowRequirementsMet else {
|
||||
sessionWarning = permissionWarningMessage()
|
||||
pipLifecycleState = .failed(.notPossible)
|
||||
traceState("startSessionAsync.blocked", extra: "reason=permissions")
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
isColdStartHandoff = false
|
||||
return
|
||||
}
|
||||
|
||||
switch await pipController.startAndWait() {
|
||||
case .started:
|
||||
activateFlowSessionAfterPiPProof(duration: duration)
|
||||
traceState("startSessionAsync.ready")
|
||||
debug("Flow session started (PiP keep-alive), mic released between utterances")
|
||||
case .failed(let failure):
|
||||
let message = AppL10n.string(failure.localizationKey)
|
||||
sessionWarning = message
|
||||
traceState("startSessionAsync.failed", extra: "reason=pipUnavailable failure=\(failure)")
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
if isColdStartHandoff {
|
||||
scheduleColdStartRecovery(duration: duration)
|
||||
let deadline = Date().addingTimeInterval(FlowPiPRecoveryPolicy.totalBudget)
|
||||
var lastFailure: FlowPiPStartFailure = .timedOut
|
||||
|
||||
for attempt in 1...FlowPiPRecoveryPolicy.maxAttempts {
|
||||
guard FlowPiPRecoveryPolicy.canContinue(
|
||||
operation: operation,
|
||||
currentOperation: pipRecoveryOperation,
|
||||
wantsActiveSession: wantsActiveSession,
|
||||
isAppForeground: isAppForeground,
|
||||
taskIsCancelled: Task.isCancelled
|
||||
) else {
|
||||
if operation == pipRecoveryOperation,
|
||||
wantsActiveSession,
|
||||
!Task.isCancelled,
|
||||
!isAppForeground {
|
||||
pipLifecycleState = .waitingForForeground
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
let delay = FlowPiPRecoveryPolicy.retryDelay(beforeAttempt: attempt)
|
||||
if delay > 0 {
|
||||
try? await Task.sleep(
|
||||
nanoseconds: UInt64(delay * 1_000_000_000)
|
||||
)
|
||||
}
|
||||
guard FlowPiPRecoveryPolicy.canContinue(
|
||||
operation: operation,
|
||||
currentOperation: pipRecoveryOperation,
|
||||
wantsActiveSession: wantsActiveSession,
|
||||
isAppForeground: isAppForeground,
|
||||
taskIsCancelled: Task.isCancelled
|
||||
) else { return }
|
||||
|
||||
if resetFirstGeneration || attempt > 1 {
|
||||
pipController.resetGeneration()
|
||||
}
|
||||
|
||||
let remaining = deadline.timeIntervalSinceNow
|
||||
guard remaining > 0 else { break }
|
||||
pipLifecycleState = isActive
|
||||
? .recovering(attempt: attempt, total: FlowPiPRecoveryPolicy.maxAttempts)
|
||||
: .preparing(attempt: attempt, total: FlowPiPRecoveryPolicy.maxAttempts)
|
||||
traceState(
|
||||
"pipRecovery.attempt",
|
||||
extra: "trigger=\(trigger.rawValue) attempt=\(attempt) generation=\(pipController.generation)"
|
||||
)
|
||||
|
||||
let outcome = await pipController.startAndWait(
|
||||
hostTimeout: FlowPiPRecoveryPolicy.hostTimeout(
|
||||
attempt: attempt,
|
||||
remainingBudget: remaining
|
||||
),
|
||||
activeTimeout: FlowPiPRecoveryPolicy.activeTimeout(
|
||||
remainingBudget: deadline.timeIntervalSinceNow
|
||||
)
|
||||
)
|
||||
guard FlowPiPRecoveryPolicy.canContinue(
|
||||
operation: operation,
|
||||
currentOperation: pipRecoveryOperation,
|
||||
wantsActiveSession: wantsActiveSession,
|
||||
isAppForeground: isAppForeground,
|
||||
taskIsCancelled: Task.isCancelled
|
||||
) else { return }
|
||||
|
||||
switch outcome {
|
||||
case .started:
|
||||
sessionWarning = nil
|
||||
pipLifecycleState = .active
|
||||
if isActive {
|
||||
refreshHostReady()
|
||||
} else {
|
||||
activateFlowSessionAfterPiPProof(duration: duration)
|
||||
}
|
||||
traceState(
|
||||
"startSessionAsync.ready",
|
||||
extra: "trigger=\(trigger.rawValue) attempt=\(attempt) generation=\(pipController.generation)"
|
||||
)
|
||||
handleColdStartAfterSessionReady()
|
||||
debug("Flow session started (PiP keep-alive), mic released between utterances")
|
||||
return
|
||||
case .failed(let failure):
|
||||
lastFailure = failure
|
||||
traceState(
|
||||
"startSessionAsync.failed",
|
||||
extra: "reason=pipUnavailable trigger=\(trigger.rawValue) attempt=\(attempt) failure=\(failure)"
|
||||
)
|
||||
}
|
||||
debug("PiP keep-alive failed to start: \(failure)")
|
||||
}
|
||||
|
||||
pipController.resetGeneration()
|
||||
pipLifecycleState = .failed(lastFailure)
|
||||
sessionWarning = AppL10n.string(lastFailure.localizationKey)
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
refreshHostReady()
|
||||
debug("PiP keep-alive failed after bounded recovery: \(lastFailure)")
|
||||
}
|
||||
|
||||
private func activateFlowSessionAfterPiPProof(duration: TimeInterval?) {
|
||||
@@ -762,6 +1039,8 @@ final class FlowSessionManager: ObservableObject {
|
||||
FlowSessionBridge.markSessionActivePersistent(sessionId: sessionId)
|
||||
FlowSessionDarwin.postSessionChanged()
|
||||
isActive = true
|
||||
wantsActiveSession = true
|
||||
pipLifecycleState = .active
|
||||
// Low-profile PiP is a system-owned keep-alive surface. Keeping the
|
||||
// display awake wastes far more power than the static PiP itself.
|
||||
ScreenWakeLock.release()
|
||||
@@ -862,18 +1141,10 @@ final class FlowSessionManager: ObservableObject {
|
||||
guard isColdStartHandoff, isActive else { return }
|
||||
sessionWarning = nil
|
||||
if !pipController.isPictureInPictureActive {
|
||||
switch await pipController.startAndWait() {
|
||||
case .started:
|
||||
break
|
||||
case .failed(let failure):
|
||||
let message = AppL10n.string(failure.localizationKey)
|
||||
sessionWarning = message
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
scheduleColdStartRecovery(duration: nil)
|
||||
debug("existing PiP session failed cold-start restart: \(failure)")
|
||||
return
|
||||
}
|
||||
ensurePiPActive(trigger: .existingSession, duration: nil)
|
||||
return
|
||||
}
|
||||
pipLifecycleState = .active
|
||||
refreshHostReady()
|
||||
handleColdStartAfterSessionReady()
|
||||
}
|
||||
@@ -895,7 +1166,9 @@ final class FlowSessionManager: ObservableObject {
|
||||
let message = AppL10n.string("flow.session.error.notPossible")
|
||||
sessionWarning = message
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
scheduleColdStartRecovery(duration: nil)
|
||||
if !pipController.isPictureInPictureActive {
|
||||
scheduleColdStartRecovery(duration: nil)
|
||||
}
|
||||
debug("cold-start blocked: host ready contract not published")
|
||||
return
|
||||
}
|
||||
@@ -927,31 +1200,14 @@ final class FlowSessionManager: ObservableObject {
|
||||
/// bounce the audio session and rebuild. Force-quit relaunches routinely
|
||||
/// inherit stale mediaserverd state that only a rebuild clears.
|
||||
private func scheduleColdStartRecovery(duration: TimeInterval?) {
|
||||
coldStartRecoveryTask?.cancel()
|
||||
coldStartRecoveryTask = Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
// The failed controller has been fully torn down. Give the app
|
||||
// scene and Pegasus service a short settling interval before the
|
||||
// one bounded cold-start retry creates a new controller generation.
|
||||
try? await Task.sleep(nanoseconds: 400_000_000)
|
||||
guard !Task.isCancelled, self.isColdStartHandoff else { return }
|
||||
let outcome = await self.pipController.startAndWait()
|
||||
self.traceState("coldStartRecovery.pip", extra: "outcome=\(outcome)")
|
||||
guard !Task.isCancelled, self.isColdStartHandoff else { return }
|
||||
switch outcome {
|
||||
case .started:
|
||||
if self.isActive {
|
||||
self.sessionWarning = nil
|
||||
self.refreshHostReady()
|
||||
self.handleColdStartAfterSessionReady()
|
||||
} else {
|
||||
self.activateFlowSessionAfterPiPProof(duration: duration)
|
||||
self.handleColdStartAfterSessionReady()
|
||||
}
|
||||
case .failed(let failure):
|
||||
self.sessionWarning = AppL10n.string(failure.localizationKey)
|
||||
}
|
||||
}
|
||||
// The failed controller has been fully torn down. Give the app
|
||||
// scene and Pegasus service a short settling interval before the
|
||||
// one bounded cold-start retry creates a new controller generation.
|
||||
ensurePiPActive(
|
||||
trigger: .startupRecovery,
|
||||
duration: duration,
|
||||
forceReset: true
|
||||
)
|
||||
}
|
||||
|
||||
private func bindSessionASRIfNeeded(force: Bool = false) {
|
||||
@@ -1245,6 +1501,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
)
|
||||
)
|
||||
currentUtteranceMode = command.resolvedUtteranceMode
|
||||
pendingManagedRequestPurpose = command.managedRequestPurpose
|
||||
pendingAIConversationID = currentUtteranceMode == .aiQuestion
|
||||
? command.aiConversationID
|
||||
: nil
|
||||
@@ -1357,6 +1614,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
commandSeq: command.commandSeq
|
||||
) else { return }
|
||||
currentUtteranceMode = .aiQuestion
|
||||
pendingManagedRequestPurpose = command.managedRequestPurpose
|
||||
pendingAIConversationID = command.aiConversationID
|
||||
pendingEditSourceText = nil
|
||||
pendingSourceHistoryEntryID = nil
|
||||
@@ -1435,7 +1693,9 @@ final class FlowSessionManager: ObservableObject {
|
||||
store: pipelineStore,
|
||||
conversations: aiConversations,
|
||||
taskKind: request.taskKind,
|
||||
thinkingEnabled: request.thinkingEnabled
|
||||
thinkingEnabled: request.thinkingEnabled,
|
||||
analyticsClient: analyticsClient,
|
||||
analyticsFeature: Self.analyticsFeature(for: request.taskKind)
|
||||
)
|
||||
aiAnswerStreamThrottle = AIAnswerStreamThrottle()
|
||||
let answer = try await service.answer(
|
||||
@@ -1610,6 +1870,13 @@ final class FlowSessionManager: ObservableObject {
|
||||
kind: FlowSessionKeys.TranscriptionErrorKind = .generic,
|
||||
status: FlowResult.Status = .error
|
||||
) {
|
||||
finishTranscriptionAnalytics(
|
||||
success: false,
|
||||
failureCategory: Self.transcriptionFailureCategory(
|
||||
kind: kind,
|
||||
status: status
|
||||
)
|
||||
)
|
||||
guard let activeSessionId, let currentUtteranceId else { return }
|
||||
FlowSessionBridge.writeResult(
|
||||
FlowResult(
|
||||
@@ -1901,6 +2168,11 @@ final class FlowSessionManager: ObservableObject {
|
||||
)
|
||||
|
||||
let cloudASRForStreaming = useStreaming ? cloudService : nil
|
||||
transcriptionAnalyticsOperation?.cancel()
|
||||
transcriptionAnalyticsOperation = analyticsClient.startAIFeature(
|
||||
.transcription,
|
||||
executionMode: analyticsExecutionMode
|
||||
)
|
||||
|
||||
asrTask = Task.detached(priority: .userInitiated) { [weak manager = self] in
|
||||
let outcome: ChunkedUtterancePipelineOutcome
|
||||
@@ -2220,6 +2492,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
pendingSourceHistoryEntryID = nil
|
||||
pendingSourceHistoryEntryRevision = nil
|
||||
pendingAIConversationID = nil
|
||||
pendingManagedRequestPurpose = nil
|
||||
pendingProcessingDeadlineAt = nil
|
||||
currentUtteranceMode = .dictation
|
||||
}
|
||||
@@ -2237,6 +2510,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
let sourceHistoryEntryID = pendingSourceHistoryEntryID
|
||||
let sourceHistoryEntryRevision = pendingSourceHistoryEntryRevision
|
||||
let aiConversationID = pendingAIConversationID
|
||||
let managedRequestPurpose = pendingManagedRequestPurpose
|
||||
let processingDeadlineAt = pendingProcessingDeadlineAt
|
||||
let asrEngineMode = sessionASREngineMode ?? store.engineMode
|
||||
// ALWAYS clear the processing gate for this utterance. The previous
|
||||
@@ -2250,6 +2524,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
pendingSourceHistoryEntryID = nil
|
||||
pendingSourceHistoryEntryRevision = nil
|
||||
pendingAIConversationID = nil
|
||||
pendingManagedRequestPurpose = nil
|
||||
pendingProcessingDeadlineAt = nil
|
||||
if currentUtteranceMode == utteranceMode {
|
||||
currentUtteranceMode = .dictation
|
||||
@@ -2366,6 +2641,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
)
|
||||
return
|
||||
}
|
||||
finishTranscriptionAnalytics(success: true)
|
||||
|
||||
// Re-read App Group at finalize so dictionary and translation changes
|
||||
// from the keyboard extension are visible before local correction,
|
||||
@@ -2427,7 +2703,9 @@ final class FlowSessionManager: ObservableObject {
|
||||
let service = try AIQuestionService.configured(
|
||||
store: pipelineStore,
|
||||
conversations: aiConversations,
|
||||
taskKind: .aiQuestion
|
||||
taskKind: .aiQuestion,
|
||||
analyticsClient: analyticsClient,
|
||||
analyticsFeature: .aiAssistant
|
||||
)
|
||||
aiAnswerStreamThrottle = AIAnswerStreamThrottle()
|
||||
let publishUtteranceId = finalizeUtteranceId
|
||||
@@ -2564,6 +2842,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
systemPrompt: instructionPrompt?.system,
|
||||
providerIdOverride: pipelineStore.polishProviderIdOverride,
|
||||
taskKind: isEditLastInput ? .editLastInput : nil,
|
||||
requestPurpose: managedRequestPurpose,
|
||||
context: isInstructionMode ? nil : polishContext,
|
||||
timeoutLimit: processingDeadlineAt.map {
|
||||
max(0.1, $0 - Date().timeIntervalSince1970)
|
||||
@@ -2868,6 +3147,13 @@ final class FlowSessionManager: ObservableObject {
|
||||
status: FlowResult.Status = .error,
|
||||
aiConversationID: UUID? = nil
|
||||
) {
|
||||
finishTranscriptionAnalytics(
|
||||
success: false,
|
||||
failureCategory: Self.transcriptionFailureCategory(
|
||||
kind: kind,
|
||||
status: status
|
||||
)
|
||||
)
|
||||
guard let sessionId, let utteranceId else { return }
|
||||
FlowTrace.warn(
|
||||
"host.deliveredError",
|
||||
@@ -2892,6 +3178,59 @@ final class FlowSessionManager: ObservableObject {
|
||||
)
|
||||
}
|
||||
|
||||
private static func analyticsFeature(
|
||||
for taskKind: ManagedGatewayTaskKind
|
||||
) -> AnalyticsFeature {
|
||||
switch taskKind {
|
||||
case .aiQuestion:
|
||||
return .hotword
|
||||
case .clipboardTransform, .customSkill, .agentPlanning:
|
||||
return .agent
|
||||
case .dictationPolish, .translation, .editLastInput:
|
||||
return .other
|
||||
}
|
||||
}
|
||||
|
||||
private var analyticsExecutionMode: AnalyticsExecutionMode {
|
||||
if store.engineMode == "local" {
|
||||
return .local
|
||||
}
|
||||
return store.credentialSource == .managed ? .managed : .byok
|
||||
}
|
||||
|
||||
private func finishTranscriptionAnalytics(
|
||||
success: Bool,
|
||||
failureCategory: AnalyticsFailureCategory = .unknown
|
||||
) {
|
||||
guard let operation = transcriptionAnalyticsOperation else { return }
|
||||
transcriptionAnalyticsOperation = nil
|
||||
if success {
|
||||
operation.succeed()
|
||||
} else {
|
||||
operation.fail(category: failureCategory)
|
||||
}
|
||||
}
|
||||
|
||||
private static func transcriptionFailureCategory(
|
||||
kind: FlowSessionKeys.TranscriptionErrorKind,
|
||||
status: FlowResult.Status
|
||||
) -> AnalyticsFailureCategory {
|
||||
if status == .aborted {
|
||||
return .cancelled
|
||||
}
|
||||
if status == .timeout {
|
||||
return .timeout
|
||||
}
|
||||
switch kind {
|
||||
case .noSpeech, .discardedEmpty:
|
||||
return .validation
|
||||
case .recognitionInterrupted:
|
||||
return .cancelled
|
||||
case .asrFailed, .audioUnavailable, .generic:
|
||||
return .provider
|
||||
}
|
||||
}
|
||||
|
||||
private static var lastResultRevision: Int64 = 0
|
||||
|
||||
private static func resultRevision() -> Int64 {
|
||||
@@ -3030,6 +3369,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
systemPrompt: String? = nil,
|
||||
providerIdOverride: String?,
|
||||
taskKind: ManagedGatewayTaskKind? = nil,
|
||||
requestPurpose: ManagedGatewayRequestPurpose? = nil,
|
||||
context: PolishContext?,
|
||||
timeoutLimit: TimeInterval? = nil
|
||||
) async throws -> PolishingService.PolishOutcome {
|
||||
@@ -3042,6 +3382,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
systemPrompt: systemPrompt,
|
||||
providerIdOverride: providerIdOverride,
|
||||
taskKind: taskKind,
|
||||
requestPurpose: requestPurpose,
|
||||
context: context
|
||||
)
|
||||
}
|
||||
@@ -3081,6 +3422,14 @@ final class FlowSessionManager: ObservableObject {
|
||||
self.isUtteranceRecording || self.isUtteranceProcessing {
|
||||
await self.reactivateCaptureIfNeeded()
|
||||
}
|
||||
if self.wantsActiveSession,
|
||||
!self.pipController.isPictureInPictureActive {
|
||||
if self.isAppForeground {
|
||||
self.ensurePiPActive(trigger: .healthCheck, duration: nil)
|
||||
} else {
|
||||
self.pipLifecycleState = .waitingForForeground
|
||||
}
|
||||
}
|
||||
FlowSessionBridge.writeHeartbeat()
|
||||
self.refreshHostReady()
|
||||
try? await Task.sleep(nanoseconds: 1_000_000_000)
|
||||
@@ -3123,6 +3472,9 @@ final class FlowSessionManager: ObservableObject {
|
||||
"event=\(event)",
|
||||
"active=\(isActive)",
|
||||
"starting=\(isStarting)",
|
||||
"wantsActive=\(wantsActiveSession)",
|
||||
"pipState=\(String(describing: pipLifecycleState))",
|
||||
"pipGeneration=\(pipController.generation)",
|
||||
"coldStart=\(isColdStartHandoff)",
|
||||
"sessionId=\(sessionId)",
|
||||
"utteranceId=\(utteranceId)",
|
||||
|
||||
Reference in New Issue
Block a user