feat(app): add resilient Flow recovery and privacy-safe analytics
CI / Validate manifests (push) Has been cancelled
CI / SwiftLint (push) Has been cancelled
CI / iOS / Extension (push) Has been cancelled
CI / macOS (push) Has been cancelled

Unify PiP recovery across startup and foreground transitions, surface actionable status, and add privacy-safe analytics plus the refreshed onboarding and account experience. Update documentation assets and advance the release build to 84.
This commit is contained in:
Rocky
2026-08-20 20:22:20 +08:00
parent ee7de5c934
commit 10bc457c72
69 changed files with 7891 additions and 598 deletions
+440 -88
View File
@@ -31,6 +31,67 @@ enum FlowUtteranceLifecyclePolicy {
}
}
enum FlowPiPActivationTrigger: String, Sendable {
case coldStart
case foreground
case existingSession
case startupRecovery
case unexpectedStop
case healthCheck
case manualRetry
}
enum FlowPiPReconciliationDecision: Equatable, Sendable {
case noSessionIntent
case alreadyActive
case waitForForeground
case startRecovery
}
enum FlowPiPRecoveryPolicy {
static let maxAttempts = 3
static let totalBudget: TimeInterval = 5
static func reconciliationDecision(
wantsActiveSession: Bool,
isAppForeground: Bool,
isPictureInPictureActive: Bool
) -> FlowPiPReconciliationDecision {
guard wantsActiveSession else { return .noSessionIntent }
if isPictureInPictureActive { return .alreadyActive }
return isAppForeground ? .startRecovery : .waitForForeground
}
static func canContinue(
operation: UInt64,
currentOperation: UInt64,
wantsActiveSession: Bool,
isAppForeground: Bool,
taskIsCancelled: Bool
) -> Bool {
operation == currentOperation
&& wantsActiveSession
&& isAppForeground
&& !taskIsCancelled
}
static func retryDelay(beforeAttempt attempt: Int) -> TimeInterval {
switch attempt {
case 1: return 0
case 2: return 0.25
default: return 0.6
}
}
static func hostTimeout(attempt: Int, remainingBudget: TimeInterval) -> TimeInterval {
min(attempt == 1 ? 1.5 : 0.25, max(0, remainingBudget))
}
static func activeTimeout(remainingBudget: TimeInterval) -> TimeInterval {
min(1.4, max(0, remainingBudget))
}
}
@MainActor
final class FlowSessionManager: ObservableObject {
private struct PrefilledAIRequest {
@@ -45,12 +106,14 @@ final class FlowSessionManager: ObservableObject {
@Published private(set) var sessionExpiresAt: Date?
/// Non-nil when continuous capture failed or permissions are missing.
@Published private(set) var sessionWarning: String?
@Published private(set) var pipLifecycleState: FlowPiPLifecycleState = .inactive
private let capture = FlowContinuousCapture()
private let pipController = FlowPictureInPictureController()
private let pipController: any FlowPictureInPictureControlling
private let store = AppGroupStore()
private let analyticsClient: any AnalyticsClient
/// Cloud / local polish via the user-configured LLM provider.
private let polisher = PolishingService()
private let polisher: PolishingService
/// AI-mode turns are intentionally process-local and never persisted.
private let aiConversations = AIConversationStore()
/// Cached ASR instance. The only on-device backend is iOS
@@ -98,6 +161,7 @@ final class FlowSessionManager: ObservableObject {
private var pendingSourceHistoryEntryID: UUID?
private var pendingSourceHistoryEntryRevision: Int64?
private var pendingAIConversationID: UUID?
private var pendingManagedRequestPurpose: ManagedGatewayRequestPurpose?
private var pendingProcessingDeadlineAt: TimeInterval?
private var pendingStopUtteranceId: UUID?
private var currentCommandSeq: Int64 = 0
@@ -110,6 +174,7 @@ final class FlowSessionManager: ObservableObject {
@Published private(set) var isUtteranceProcessing = false
private var finalizeTask: Task<Void, Never>?
private var asrTask: Task<Void, Never>?
private var transcriptionAnalyticsOperation: (any AnalyticsAIOperation)?
private var utteranceSafetyTask: Task<Void, Never>?
private var chunkedPipeline: ChunkedUtterancePipeline?
private var currentPartial = ""
@@ -131,7 +196,10 @@ final class FlowSessionManager: ObservableObject {
private var isAppForeground = false
/// True while handling a keyboard-initiated `startflow` cold start.
private var isColdStartHandoff = false
private var coldStartRecoveryTask: Task<Void, Never>?
/// User intent outlives a transient PiP loss; only explicit teardown clears it.
private var wantsActiveSession = false
/// Invalidates late async completions after retry, stop, or process teardown.
private var pipRecoveryOperation: UInt64 = 0
var shouldDeferHostHeavyWork: Bool {
isStarting
|| startingUtteranceId != nil
@@ -148,7 +216,13 @@ final class FlowSessionManager: ObservableObject {
/// recreate the `@StateObject`-owned manager within the same process).
private static var didRunLaunchReconciliation = false
init() {
init(
analyticsClient: any AnalyticsClient = NoopAnalyticsClient(),
pipController: any FlowPictureInPictureControlling = FlowPictureInPictureController()
) {
self.analyticsClient = analyticsClient
self.polisher = PolishingService(analyticsClient: analyticsClient)
self.pipController = pipController
// Sessions are (re)started explicitly on app foreground via
// `activateOnForeground()`. We deliberately do NOT silently reattach a
// stored session here — after a force-quit that would resurrect capture
@@ -194,10 +268,9 @@ final class FlowSessionManager: ObservableObject {
kind: .recognitionInterrupted
)
}
pipController.onUserDismissed = { [weak self] in
guard let self, self.isActive else { return }
self.debug("PiP dismissed by user — ending Flow session")
self.endSession()
pipController.onUnexpectedStop = { [weak self] in
guard let self, self.wantsActiveSession else { return }
self.handleUnexpectedPiPStop()
}
FlowTerminationCoordinator.register(self)
}
@@ -257,25 +330,25 @@ final class FlowSessionManager: ObservableObject {
}
if isActive {
wantsActiveSession = true
extendSession(duration: duration)
refreshHostReady()
if pipController.isPictureInPictureActive {
pipLifecycleState = .active
refreshHostReady()
} else {
ensurePiPActive(
trigger: coldStart ? .coldStart : .existingSession,
duration: duration
)
}
return
}
guard !isStarting else {
traceState("startSession.ignored", extra: "reason=alreadyStarting")
return
}
// Claim the flag synchronously: on a cold start the URL router and
// `activateOnForeground()` both fire in the same runloop turn, and
// setting it inside the async body let two start bodies interleave.
isStarting = true
startTask?.cancel()
startTask = Task { @MainActor [weak self] in
await self?.startSessionAsync(duration: duration)
self?.handleColdStartAfterSessionReady()
}
wantsActiveSession = true
ensurePiPActive(
trigger: coldStart ? .coldStart : .foreground,
duration: duration
)
}
/// Clears App Group Flow state left behind when the host process was killed
@@ -339,9 +412,19 @@ final class FlowSessionManager: ObservableObject {
)
}
/// User-requested recovery always discards the rejected AVKit generation.
func retryPiPRecovery() {
guard AppGroup.isAvailable else { return }
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
FlowSessionBridge.setHostReady(false)
return
}
wantsActiveSession = true
ensurePiPActive(trigger: .manualRetry, duration: nil, forceReset: true)
}
private func completeColdStartHandoff() {
coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = nil
isColdStartHandoff = false
if isActive {
refreshHostReady()
@@ -360,9 +443,10 @@ final class FlowSessionManager: ObservableObject {
)
}
wantsActiveSession = false
pipRecoveryOperation &+= 1
pipLifecycleState = .inactive
isColdStartHandoff = false
coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = nil
startTask?.cancel()
startTask = nil
startupAudioHealthTask?.cancel()
@@ -433,7 +517,7 @@ final class FlowSessionManager: ObservableObject {
}
func endSession() {
guard isActive else { return }
guard isActive || isStarting || wantsActiveSession else { return }
debug("Flow session ended")
if isUtteranceRecording || isUtteranceProcessing,
claimTerminal(utteranceId: currentUtteranceId) {
@@ -444,9 +528,10 @@ final class FlowSessionManager: ObservableObject {
)
}
wantsActiveSession = false
pipRecoveryOperation &+= 1
pipLifecycleState = .inactive
isColdStartHandoff = false
coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = nil
startTask?.cancel()
startTask = nil
startupAudioHealthTask?.cancel()
@@ -493,6 +578,7 @@ final class FlowSessionManager: ObservableObject {
FlowSessionBridge.markSessionInactive()
FlowSessionDarwin.postSessionChanged()
isActive = false
isStarting = false
sessionExpiresAt = nil
sessionWarning = nil
currentPartial = ""
@@ -516,19 +602,27 @@ final class FlowSessionManager: ObservableObject {
case .active:
setAppForeground(true)
resumeAfterForeground()
if wantsActiveSession, !pipController.isPictureInPictureActive {
ensurePiPActive(trigger: .foreground, duration: nil)
}
case .inactive:
setAppForeground(false)
writeHeartbeatIfActive()
if isActive {
if isActive, pipController.isPictureInPictureActive {
Task { @MainActor [weak self] in
await self?.pipController.prepareForBackgroundAutoStart()
}
} else if wantsActiveSession {
pausePiPRecoveryUntilForeground()
}
case .background:
setAppForeground(false)
if isActive {
if isActive, pipController.isPictureInPictureActive {
Task { @MainActor [weak self] in
await self?.pipController.prepareForBackgroundAutoStart()
}
} else if wantsActiveSession {
pausePiPRecoveryUntilForeground()
}
if isColdStartHandoff {
completeColdStartHandoff()
@@ -549,6 +643,33 @@ final class FlowSessionManager: ObservableObject {
FlowSessionBridge.writeHeartbeat()
}
private func handleUnexpectedPiPStop() {
guard wantsActiveSession else { return }
traceState("pip.unexpectedStop")
pipLifecycleState = .waitingForForeground
sessionWarning = nil
if isUtteranceRecording {
failUtterance(
message: AppL10n.string("flow.error.recognitionInterrupted"),
kind: .recognitionInterrupted
)
}
FlowSessionBridge.setHostReady(false)
refreshHostReady()
// Do not immediately fight another app for the system-owned PiP slot.
// The scene-active path or the existing heartbeat performs reconciliation.
}
private func pausePiPRecoveryUntilForeground() {
startTask?.cancel()
startTask = nil
pipRecoveryOperation &+= 1
pipController.resetGeneration()
isStarting = false
pipLifecycleState = .waitingForForeground
refreshHostReady()
}
private func resumeAfterForeground() {
guard isActive else {
return
@@ -723,36 +844,192 @@ final class FlowSessionManager: ObservableObject {
// MARK: - Session start
private func startSessionAsync(duration: TimeInterval?) async {
// `isStarting` was claimed synchronously in `startSession()`.
defer { isStarting = false }
private func ensurePiPActive(
trigger: FlowPiPActivationTrigger,
duration: TimeInterval?,
forceReset: Bool = false
) {
switch FlowPiPRecoveryPolicy.reconciliationDecision(
wantsActiveSession: wantsActiveSession,
isAppForeground: isAppForeground,
isPictureInPictureActive: pipController.isPictureInPictureActive
) {
case .noSessionIntent:
return
case .alreadyActive:
pipLifecycleState = .active
sessionWarning = nil
if isActive {
refreshHostReady()
} else {
activateFlowSessionAfterPiPProof(duration: duration)
}
handleColdStartAfterSessionReady()
return
case .waitForForeground:
pipLifecycleState = .waitingForForeground
FlowSessionBridge.setHostReady(false)
refreshHostReady()
return
case .startRecovery:
break
}
guard startTask == nil else {
traceState(
"pipRecovery.coalesced",
extra: "trigger=\(trigger.rawValue)"
)
return
}
// Claim the flag synchronously: on a cold start the URL router and
// `activateOnForeground()` both fire in the same runloop turn, and
// setting it inside the async body let two start bodies interleave.
isStarting = true
pipRecoveryOperation &+= 1
let operation = pipRecoveryOperation
let recoveringExistingSession = isActive
let resetFirstGeneration = forceReset
|| recoveringExistingSession
|| pipController.generation > 0
pipLifecycleState = recoveringExistingSession
? .recovering(attempt: 1, total: FlowPiPRecoveryPolicy.maxAttempts)
: .preparing(attempt: 1, total: FlowPiPRecoveryPolicy.maxAttempts)
startTask = Task { @MainActor [weak self] in
await self?.startSessionAsync(
duration: duration,
trigger: trigger,
operation: operation,
resetFirstGeneration: resetFirstGeneration
)
}
}
private func startSessionAsync(
duration: TimeInterval?,
trigger: FlowPiPActivationTrigger,
operation: UInt64,
resetFirstGeneration: Bool
) async {
// `isStarting` was claimed synchronously in `ensurePiPActive()`.
defer {
if operation == pipRecoveryOperation {
isStarting = false
startTask = nil
}
}
guard !Task.isCancelled else { return }
traceState("startSessionAsync.begin")
traceState("startSessionAsync.begin", extra: "trigger=\(trigger.rawValue)")
sessionWarning = nil
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
pipLifecycleState = .failed(.notPossible)
traceState("startSessionAsync.blocked", extra: "reason=permissions")
FlowSessionBridge.setHostReady(false)
isColdStartHandoff = false
return
}
switch await pipController.startAndWait() {
case .started:
activateFlowSessionAfterPiPProof(duration: duration)
traceState("startSessionAsync.ready")
debug("Flow session started (PiP keep-alive), mic released between utterances")
case .failed(let failure):
let message = AppL10n.string(failure.localizationKey)
sessionWarning = message
traceState("startSessionAsync.failed", extra: "reason=pipUnavailable failure=\(failure)")
FlowSessionBridge.setHostReady(false)
if isColdStartHandoff {
scheduleColdStartRecovery(duration: duration)
let deadline = Date().addingTimeInterval(FlowPiPRecoveryPolicy.totalBudget)
var lastFailure: FlowPiPStartFailure = .timedOut
for attempt in 1...FlowPiPRecoveryPolicy.maxAttempts {
guard FlowPiPRecoveryPolicy.canContinue(
operation: operation,
currentOperation: pipRecoveryOperation,
wantsActiveSession: wantsActiveSession,
isAppForeground: isAppForeground,
taskIsCancelled: Task.isCancelled
) else {
if operation == pipRecoveryOperation,
wantsActiveSession,
!Task.isCancelled,
!isAppForeground {
pipLifecycleState = .waitingForForeground
FlowSessionBridge.setHostReady(false)
}
return
}
let delay = FlowPiPRecoveryPolicy.retryDelay(beforeAttempt: attempt)
if delay > 0 {
try? await Task.sleep(
nanoseconds: UInt64(delay * 1_000_000_000)
)
}
guard FlowPiPRecoveryPolicy.canContinue(
operation: operation,
currentOperation: pipRecoveryOperation,
wantsActiveSession: wantsActiveSession,
isAppForeground: isAppForeground,
taskIsCancelled: Task.isCancelled
) else { return }
if resetFirstGeneration || attempt > 1 {
pipController.resetGeneration()
}
let remaining = deadline.timeIntervalSinceNow
guard remaining > 0 else { break }
pipLifecycleState = isActive
? .recovering(attempt: attempt, total: FlowPiPRecoveryPolicy.maxAttempts)
: .preparing(attempt: attempt, total: FlowPiPRecoveryPolicy.maxAttempts)
traceState(
"pipRecovery.attempt",
extra: "trigger=\(trigger.rawValue) attempt=\(attempt) generation=\(pipController.generation)"
)
let outcome = await pipController.startAndWait(
hostTimeout: FlowPiPRecoveryPolicy.hostTimeout(
attempt: attempt,
remainingBudget: remaining
),
activeTimeout: FlowPiPRecoveryPolicy.activeTimeout(
remainingBudget: deadline.timeIntervalSinceNow
)
)
guard FlowPiPRecoveryPolicy.canContinue(
operation: operation,
currentOperation: pipRecoveryOperation,
wantsActiveSession: wantsActiveSession,
isAppForeground: isAppForeground,
taskIsCancelled: Task.isCancelled
) else { return }
switch outcome {
case .started:
sessionWarning = nil
pipLifecycleState = .active
if isActive {
refreshHostReady()
} else {
activateFlowSessionAfterPiPProof(duration: duration)
}
traceState(
"startSessionAsync.ready",
extra: "trigger=\(trigger.rawValue) attempt=\(attempt) generation=\(pipController.generation)"
)
handleColdStartAfterSessionReady()
debug("Flow session started (PiP keep-alive), mic released between utterances")
return
case .failed(let failure):
lastFailure = failure
traceState(
"startSessionAsync.failed",
extra: "reason=pipUnavailable trigger=\(trigger.rawValue) attempt=\(attempt) failure=\(failure)"
)
}
debug("PiP keep-alive failed to start: \(failure)")
}
pipController.resetGeneration()
pipLifecycleState = .failed(lastFailure)
sessionWarning = AppL10n.string(lastFailure.localizationKey)
FlowSessionBridge.setHostReady(false)
refreshHostReady()
debug("PiP keep-alive failed after bounded recovery: \(lastFailure)")
}
private func activateFlowSessionAfterPiPProof(duration: TimeInterval?) {
@@ -762,6 +1039,8 @@ final class FlowSessionManager: ObservableObject {
FlowSessionBridge.markSessionActivePersistent(sessionId: sessionId)
FlowSessionDarwin.postSessionChanged()
isActive = true
wantsActiveSession = true
pipLifecycleState = .active
// Low-profile PiP is a system-owned keep-alive surface. Keeping the
// display awake wastes far more power than the static PiP itself.
ScreenWakeLock.release()
@@ -862,18 +1141,10 @@ final class FlowSessionManager: ObservableObject {
guard isColdStartHandoff, isActive else { return }
sessionWarning = nil
if !pipController.isPictureInPictureActive {
switch await pipController.startAndWait() {
case .started:
break
case .failed(let failure):
let message = AppL10n.string(failure.localizationKey)
sessionWarning = message
FlowSessionBridge.setHostReady(false)
scheduleColdStartRecovery(duration: nil)
debug("existing PiP session failed cold-start restart: \(failure)")
return
}
ensurePiPActive(trigger: .existingSession, duration: nil)
return
}
pipLifecycleState = .active
refreshHostReady()
handleColdStartAfterSessionReady()
}
@@ -895,7 +1166,9 @@ final class FlowSessionManager: ObservableObject {
let message = AppL10n.string("flow.session.error.notPossible")
sessionWarning = message
FlowSessionBridge.setHostReady(false)
scheduleColdStartRecovery(duration: nil)
if !pipController.isPictureInPictureActive {
scheduleColdStartRecovery(duration: nil)
}
debug("cold-start blocked: host ready contract not published")
return
}
@@ -927,31 +1200,14 @@ final class FlowSessionManager: ObservableObject {
/// bounce the audio session and rebuild. Force-quit relaunches routinely
/// inherit stale mediaserverd state that only a rebuild clears.
private func scheduleColdStartRecovery(duration: TimeInterval?) {
coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = Task { @MainActor [weak self] in
guard let self else { return }
// The failed controller has been fully torn down. Give the app
// scene and Pegasus service a short settling interval before the
// one bounded cold-start retry creates a new controller generation.
try? await Task.sleep(nanoseconds: 400_000_000)
guard !Task.isCancelled, self.isColdStartHandoff else { return }
let outcome = await self.pipController.startAndWait()
self.traceState("coldStartRecovery.pip", extra: "outcome=\(outcome)")
guard !Task.isCancelled, self.isColdStartHandoff else { return }
switch outcome {
case .started:
if self.isActive {
self.sessionWarning = nil
self.refreshHostReady()
self.handleColdStartAfterSessionReady()
} else {
self.activateFlowSessionAfterPiPProof(duration: duration)
self.handleColdStartAfterSessionReady()
}
case .failed(let failure):
self.sessionWarning = AppL10n.string(failure.localizationKey)
}
}
// The failed controller has been fully torn down. Give the app
// scene and Pegasus service a short settling interval before the
// one bounded cold-start retry creates a new controller generation.
ensurePiPActive(
trigger: .startupRecovery,
duration: duration,
forceReset: true
)
}
private func bindSessionASRIfNeeded(force: Bool = false) {
@@ -1245,6 +1501,7 @@ final class FlowSessionManager: ObservableObject {
)
)
currentUtteranceMode = command.resolvedUtteranceMode
pendingManagedRequestPurpose = command.managedRequestPurpose
pendingAIConversationID = currentUtteranceMode == .aiQuestion
? command.aiConversationID
: nil
@@ -1357,6 +1614,7 @@ final class FlowSessionManager: ObservableObject {
commandSeq: command.commandSeq
) else { return }
currentUtteranceMode = .aiQuestion
pendingManagedRequestPurpose = command.managedRequestPurpose
pendingAIConversationID = command.aiConversationID
pendingEditSourceText = nil
pendingSourceHistoryEntryID = nil
@@ -1435,7 +1693,9 @@ final class FlowSessionManager: ObservableObject {
store: pipelineStore,
conversations: aiConversations,
taskKind: request.taskKind,
thinkingEnabled: request.thinkingEnabled
thinkingEnabled: request.thinkingEnabled,
analyticsClient: analyticsClient,
analyticsFeature: Self.analyticsFeature(for: request.taskKind)
)
aiAnswerStreamThrottle = AIAnswerStreamThrottle()
let answer = try await service.answer(
@@ -1610,6 +1870,13 @@ final class FlowSessionManager: ObservableObject {
kind: FlowSessionKeys.TranscriptionErrorKind = .generic,
status: FlowResult.Status = .error
) {
finishTranscriptionAnalytics(
success: false,
failureCategory: Self.transcriptionFailureCategory(
kind: kind,
status: status
)
)
guard let activeSessionId, let currentUtteranceId else { return }
FlowSessionBridge.writeResult(
FlowResult(
@@ -1901,6 +2168,11 @@ final class FlowSessionManager: ObservableObject {
)
let cloudASRForStreaming = useStreaming ? cloudService : nil
transcriptionAnalyticsOperation?.cancel()
transcriptionAnalyticsOperation = analyticsClient.startAIFeature(
.transcription,
executionMode: analyticsExecutionMode
)
asrTask = Task.detached(priority: .userInitiated) { [weak manager = self] in
let outcome: ChunkedUtterancePipelineOutcome
@@ -2220,6 +2492,7 @@ final class FlowSessionManager: ObservableObject {
pendingSourceHistoryEntryID = nil
pendingSourceHistoryEntryRevision = nil
pendingAIConversationID = nil
pendingManagedRequestPurpose = nil
pendingProcessingDeadlineAt = nil
currentUtteranceMode = .dictation
}
@@ -2237,6 +2510,7 @@ final class FlowSessionManager: ObservableObject {
let sourceHistoryEntryID = pendingSourceHistoryEntryID
let sourceHistoryEntryRevision = pendingSourceHistoryEntryRevision
let aiConversationID = pendingAIConversationID
let managedRequestPurpose = pendingManagedRequestPurpose
let processingDeadlineAt = pendingProcessingDeadlineAt
let asrEngineMode = sessionASREngineMode ?? store.engineMode
// ALWAYS clear the processing gate for this utterance. The previous
@@ -2250,6 +2524,7 @@ final class FlowSessionManager: ObservableObject {
pendingSourceHistoryEntryID = nil
pendingSourceHistoryEntryRevision = nil
pendingAIConversationID = nil
pendingManagedRequestPurpose = nil
pendingProcessingDeadlineAt = nil
if currentUtteranceMode == utteranceMode {
currentUtteranceMode = .dictation
@@ -2366,6 +2641,7 @@ final class FlowSessionManager: ObservableObject {
)
return
}
finishTranscriptionAnalytics(success: true)
// Re-read App Group at finalize so dictionary and translation changes
// from the keyboard extension are visible before local correction,
@@ -2427,7 +2703,9 @@ final class FlowSessionManager: ObservableObject {
let service = try AIQuestionService.configured(
store: pipelineStore,
conversations: aiConversations,
taskKind: .aiQuestion
taskKind: .aiQuestion,
analyticsClient: analyticsClient,
analyticsFeature: .aiAssistant
)
aiAnswerStreamThrottle = AIAnswerStreamThrottle()
let publishUtteranceId = finalizeUtteranceId
@@ -2564,6 +2842,7 @@ final class FlowSessionManager: ObservableObject {
systemPrompt: instructionPrompt?.system,
providerIdOverride: pipelineStore.polishProviderIdOverride,
taskKind: isEditLastInput ? .editLastInput : nil,
requestPurpose: managedRequestPurpose,
context: isInstructionMode ? nil : polishContext,
timeoutLimit: processingDeadlineAt.map {
max(0.1, $0 - Date().timeIntervalSince1970)
@@ -2868,6 +3147,13 @@ final class FlowSessionManager: ObservableObject {
status: FlowResult.Status = .error,
aiConversationID: UUID? = nil
) {
finishTranscriptionAnalytics(
success: false,
failureCategory: Self.transcriptionFailureCategory(
kind: kind,
status: status
)
)
guard let sessionId, let utteranceId else { return }
FlowTrace.warn(
"host.deliveredError",
@@ -2892,6 +3178,59 @@ final class FlowSessionManager: ObservableObject {
)
}
private static func analyticsFeature(
for taskKind: ManagedGatewayTaskKind
) -> AnalyticsFeature {
switch taskKind {
case .aiQuestion:
return .hotword
case .clipboardTransform, .customSkill, .agentPlanning:
return .agent
case .dictationPolish, .translation, .editLastInput:
return .other
}
}
private var analyticsExecutionMode: AnalyticsExecutionMode {
if store.engineMode == "local" {
return .local
}
return store.credentialSource == .managed ? .managed : .byok
}
private func finishTranscriptionAnalytics(
success: Bool,
failureCategory: AnalyticsFailureCategory = .unknown
) {
guard let operation = transcriptionAnalyticsOperation else { return }
transcriptionAnalyticsOperation = nil
if success {
operation.succeed()
} else {
operation.fail(category: failureCategory)
}
}
private static func transcriptionFailureCategory(
kind: FlowSessionKeys.TranscriptionErrorKind,
status: FlowResult.Status
) -> AnalyticsFailureCategory {
if status == .aborted {
return .cancelled
}
if status == .timeout {
return .timeout
}
switch kind {
case .noSpeech, .discardedEmpty:
return .validation
case .recognitionInterrupted:
return .cancelled
case .asrFailed, .audioUnavailable, .generic:
return .provider
}
}
private static var lastResultRevision: Int64 = 0
private static func resultRevision() -> Int64 {
@@ -3030,6 +3369,7 @@ final class FlowSessionManager: ObservableObject {
systemPrompt: String? = nil,
providerIdOverride: String?,
taskKind: ManagedGatewayTaskKind? = nil,
requestPurpose: ManagedGatewayRequestPurpose? = nil,
context: PolishContext?,
timeoutLimit: TimeInterval? = nil
) async throws -> PolishingService.PolishOutcome {
@@ -3042,6 +3382,7 @@ final class FlowSessionManager: ObservableObject {
systemPrompt: systemPrompt,
providerIdOverride: providerIdOverride,
taskKind: taskKind,
requestPurpose: requestPurpose,
context: context
)
}
@@ -3081,6 +3422,14 @@ final class FlowSessionManager: ObservableObject {
self.isUtteranceRecording || self.isUtteranceProcessing {
await self.reactivateCaptureIfNeeded()
}
if self.wantsActiveSession,
!self.pipController.isPictureInPictureActive {
if self.isAppForeground {
self.ensurePiPActive(trigger: .healthCheck, duration: nil)
} else {
self.pipLifecycleState = .waitingForForeground
}
}
FlowSessionBridge.writeHeartbeat()
self.refreshHostReady()
try? await Task.sleep(nanoseconds: 1_000_000_000)
@@ -3123,6 +3472,9 @@ final class FlowSessionManager: ObservableObject {
"event=\(event)",
"active=\(isActive)",
"starting=\(isStarting)",
"wantsActive=\(wantsActiveSession)",
"pipState=\(String(describing: pipLifecycleState))",
"pipGeneration=\(pipController.generation)",
"coldStart=\(isColdStartHandoff)",
"sessionId=\(sessionId)",
"utteranceId=\(utteranceId)",