rockyandClaude Opus 5
0023741455
Squashes the working tree that had accumulated on cloud-agent. Two strands of work, kept in one commit at the maintainer's request. Crash and stability (this session): - Subscribe to MetricKit so crash, hang, CPU, and disk-write diagnostics are recorded on device. Keyboard-extension memory kills are the most likely way a user loses the keyboard mid-sentence and iOS does not classify them as crashes, so they never reach App Store Connect. - Upload those reports to POST /v1/analytics/diagnostics from internal builds only (local Debug and TestFlight), reusing the analytics installation identifier and skipping entirely when analytics are off. App Store builds keep everything on device. - Shed the Chinese engine, English lexicon, and in-flight pipeline work at the 40 MiB and 48 MiB budget thresholds instead of waiting for a system memory warning that jetsam often preempts. - Install audio taps with the input node's live format through one shared adaptive resampler, so an audio-route change mid-recording no longer raises an uncatchable format-mismatch exception. Covers the dictation preview and the Mac recorder, which still used the pattern the Flow capture path had already moved away from. - Harden the librime bridge: never pass a nil string across a nonnull boundary when a user dictionary holds malformed UTF-8, null-check the engine API vector, and distrust a candidate count without its array. - Guard the expanded candidate grid against a list that shrinks while the collection view is still laying out against the previous count. - Resolve localized text without the App Group so the screen shown when that container is missing renders its message instead of crashing on launch, which is the one situation it exists to handle. - Gate CI on the App Group and Keychain wiring that the keyboard extension hard-depends on, with a self-test proving the check fails. Clipboard, reply, and semantics (pre-existing working tree): - Unify the reply center, refresh keyboard AI features, and add native event and reminder exporters. - Expand clipboard semantic routing with authored corpora and a product-blind holdout, plus the scripts that build and validate them. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2454 lines
100 KiB
Swift
2454 lines
100 KiB
Swift
// KeyboardFlowCoordinator.swift
|
|
// OSGKeyboard · Keyboard Extension
|
|
//
|
|
// Flow session start, recording, watchdogs, and result delivery handling.
|
|
|
|
import OSGKeyboardShared
|
|
import UIKit
|
|
|
|
@MainActor
|
|
final class KeyboardFlowCoordinator {
|
|
private enum FlowWatchdog {
|
|
static let pollIntervalNs: UInt64 = 200_000_000
|
|
static let startTimeout: TimeInterval = FlowSessionKeys.utteranceStartBudget
|
|
|
|
static func resultTimeout(engineMode: String) -> TimeInterval {
|
|
FlowSessionKeys.keyboardResultTimeout(engineMode: engineMode)
|
|
}
|
|
}
|
|
|
|
private enum VoiceCancellationKind {
|
|
case dictation
|
|
case aiQuestion
|
|
}
|
|
|
|
private enum ResultWatchdogPurpose {
|
|
case delivery
|
|
case cancellation
|
|
}
|
|
|
|
private let state: KeyboardState
|
|
private let textInserter: KeyboardTextInserter
|
|
private let hasFullAccess: () -> Bool
|
|
private let wakeLockView: () -> UIView?
|
|
private let openHostApp: (String) -> Void
|
|
private let detectAndStoreAppContext: () -> Void
|
|
private let fieldContextProvider: () -> FlowFieldContext?
|
|
private let scheduleAutoClearError: () -> Void
|
|
private let refreshConfigFromAppGroup: () -> Void
|
|
|
|
private var isPendingFlowStart = false
|
|
private var flowStartDeadline: TimeInterval = 0
|
|
/// Absolute cap for a cold-start jump. The soft `flowStartDeadline` is
|
|
/// pushed forward while the host keeps acknowledging it is warming, but
|
|
/// never past this — bounds the wait when the host is stuck in the
|
|
/// background and the user never brings it forward.
|
|
private var flowStartHardDeadline: TimeInterval = 0
|
|
/// One-shot guard so the "bring the app forward" hint is shown once per
|
|
/// start attempt instead of on every 200 ms watchdog poll.
|
|
private var coldStartForegroundHintShown = false
|
|
private var isFlowRecording = false
|
|
private var flowWatchdogTask: Task<Void, Never>?
|
|
private var utteranceTimerTask: Task<Void, Never>?
|
|
private var hostReadyWaitTask: Task<Void, Never>?
|
|
private var utteranceStartedAt: TimeInterval = 0
|
|
private var wasSessionActive = false
|
|
/// Last wall-clock time the host published a fresh ready contract. Used to
|
|
/// smooth over transient cross-process heartbeat read jitter so a single
|
|
/// stale sample never flashes the mic orange while the session is healthy.
|
|
private var lastHostReadyAt: TimeInterval = 0
|
|
private static let hostReadyGrace: TimeInterval = 4
|
|
/// Once the host has published ready for this session, hold green through
|
|
/// brief inter-utterance ready flaps instead of flashing preparingSession.
|
|
private var sessionProvenReady = false
|
|
private var flowSessionMonitorTask: Task<Void, Never>?
|
|
private var isAwaitingFlowResult = false
|
|
private var activeSessionId: UUID?
|
|
private var currentUtteranceId: UUID?
|
|
private var currentStartDeadlineAt: TimeInterval?
|
|
private var currentUtteranceRequest: FlowUtteranceRequest?
|
|
private var audioPrimeID: UUID?
|
|
private var audioPrimeCancelTask: Task<Void, Never>?
|
|
private var editHostConfirmed = false
|
|
private var cancelledEditUtteranceIDs: Set<UUID> = []
|
|
private var cancelledDictationUtteranceIDs: Set<UUID> = []
|
|
private var cancelledAIUtteranceIDs: Set<UUID> = []
|
|
var onEditHostRecordingConfirmed: () -> Void = {}
|
|
var onEditResult: (FlowResult) -> Void = { _ in }
|
|
var onEditFailure: (String) -> Void = { _ in }
|
|
var onAIUtterancePrepared: (UUID) -> Void = { _ in }
|
|
var onAIRecordingStarted: (UUID) -> Void = { _ in }
|
|
var onAIRecognitionStarted: (UUID) -> Void = { _ in }
|
|
var onAIGeneratingStarted: (UUID) -> Void = { _ in }
|
|
var onAITranscript: (String, UUID, FlowResult.Status) -> Void = { _, _, _ in }
|
|
var onAIStreamingAnswer: (String, UUID) -> Void = { _, _ in }
|
|
var onAIResult: (FlowResult) -> Void = { _ in }
|
|
var onAIFailure: (String, UUID?) -> Void = { _, _ in }
|
|
var onAICancelled: () -> Void = {}
|
|
/// Utterance whose final result we already inserted (or failed). Prevents
|
|
/// `adoptHostBusyStateIfNeeded` from re-entering `.processing` after a
|
|
/// stale App Group snapshot still says `reason=processing`.
|
|
private var lastConsumedUtteranceId: UUID?
|
|
/// Utterance we just asked the host to stop. Until the host publishes
|
|
/// processing/final state, stale App Group snapshots can still say
|
|
/// `reason=recording`; do not re-adopt that utterance as locally active.
|
|
private var lastStoppedUtteranceId: UUID?
|
|
private var currentCommandSeq: Int64 = 0
|
|
private var lastAvailabilityTraceSignature = ""
|
|
/// When true, `completeFlowStartHandoff` starts recording after the host
|
|
/// publishes ready — set only for an explicit mic press.
|
|
private var recordAfterHandoff = false
|
|
/// When true, `startHostReadyWaitIfNeeded` starts recording once ready
|
|
/// (mic pressed while session was still warming / mid ready-flap).
|
|
private var recordWhenHostReady = false
|
|
/// Ignores single-frame "host dead" samples before allowing a cold-start jump
|
|
/// from non-press recovery paths.
|
|
private var coldStartDebouncer = FlowColdStartDebouncer()
|
|
init(
|
|
state: KeyboardState,
|
|
textInserter: KeyboardTextInserter,
|
|
hasFullAccess: @escaping () -> Bool,
|
|
wakeLockView: @escaping () -> UIView?,
|
|
openHostApp: @escaping (String) -> Void,
|
|
detectAndStoreAppContext: @escaping () -> Void,
|
|
fieldContextProvider: @escaping () -> FlowFieldContext?,
|
|
scheduleAutoClearError: @escaping () -> Void,
|
|
refreshConfigFromAppGroup: @escaping () -> Void
|
|
) {
|
|
self.state = state
|
|
self.textInserter = textInserter
|
|
self.hasFullAccess = hasFullAccess
|
|
self.wakeLockView = wakeLockView
|
|
self.openHostApp = openHostApp
|
|
self.detectAndStoreAppContext = detectAndStoreAppContext
|
|
self.fieldContextProvider = fieldContextProvider
|
|
self.scheduleAutoClearError = scheduleAutoClearError
|
|
self.refreshConfigFromAppGroup = refreshConfigFromAppGroup
|
|
}
|
|
|
|
var preservesLifecycleOnDisappear: Bool {
|
|
isPendingFlowStart
|
|
|| isFlowRecording
|
|
|| isAwaitingFlowResult
|
|
|| currentUtteranceRequest != nil
|
|
}
|
|
|
|
private var voiceSetupReady: Bool {
|
|
state.hasCompletedOnboarding || state.isOnboardingPracticeActive
|
|
}
|
|
|
|
var isEditSessionActive: Bool { currentUtteranceRequest?.isEdit == true }
|
|
|
|
/// Session/transcription changes are pushed in real time by Darwin
|
|
/// notifications (see `KeyboardConfigSync.installDarwinObservers`), so this
|
|
/// loop is only a low-frequency safety net for coalesced/dropped Darwin
|
|
/// signals — hence 3 s rather than 1 Hz to save battery while idle.
|
|
private static let sessionMonitorIntervalNs: UInt64 = 3_000_000_000
|
|
|
|
func startSessionMonitor() {
|
|
flowSessionMonitorTask?.cancel()
|
|
flowSessionMonitorTask = Task { @MainActor [weak self] in
|
|
while !Task.isCancelled {
|
|
self?.refreshSessionState()
|
|
try? await Task.sleep(nanoseconds: Self.sessionMonitorIntervalNs)
|
|
}
|
|
}
|
|
}
|
|
|
|
func stopSessionMonitor() {
|
|
flowSessionMonitorTask?.cancel()
|
|
flowSessionMonitorTask = nil
|
|
stopHostReadyWait()
|
|
}
|
|
|
|
/// Ensure the containing app has armed its low-profile PiP even when the
|
|
/// keyboard opens directly into Pinyin/English typing mode. The mic stays
|
|
/// visually ready; if the host contract is missing, one automatic handoff
|
|
/// prepares PiP so the next press does not need another app switch.
|
|
///
|
|
/// Must **not** re-jump when a session is already alive/warming (common after
|
|
/// the user returns from startflow and taps the Voice tab — height/appear
|
|
/// noise used to call this again while `ready` briefly lagged).
|
|
func ensurePiPReadyOnKeyboardOpen() {
|
|
guard FlowHandoffPolicy.allowsProactiveHostAutoLaunch,
|
|
voiceSetupReady,
|
|
hasFullAccess(),
|
|
AppGroup.isAvailable,
|
|
!isPendingFlowStart,
|
|
!isFlowRecording,
|
|
!isAwaitingFlowResult else { return }
|
|
|
|
FlowSessionBridge.reloadFromDisk()
|
|
let withinReadyGrace = lastHostReadyAt > 0
|
|
&& (Date().timeIntervalSince1970 - lastHostReadyAt) <= Self.hostReadyGrace
|
|
let shouldArm = FlowHandoffPolicy.shouldProactivePiPArm(
|
|
hostReady: FlowSessionBridge.isHostReady(),
|
|
snapshotReason: FlowSessionBridge.readySnapshot()?.reason,
|
|
sessionActive: FlowSessionBridge.isSessionActive(),
|
|
hostReachable: FlowSessionBridge.isHostReachable(),
|
|
hostStale: FlowSessionBridge.isHostStale(),
|
|
withinReadyGrace: withinReadyGrace,
|
|
inCooldown: FlowSessionBridge.isPiPArmInCooldown(),
|
|
heartbeatStaleness: FlowSessionBridge.heartbeatStaleness()
|
|
)
|
|
guard shouldArm else {
|
|
traceState("keyboardOpen.autoArmPiP.skipped", extra: "gate=0")
|
|
return
|
|
}
|
|
|
|
FlowSessionBridge.markPiPArmAttempt()
|
|
detectAndStoreAppContext()
|
|
beginFlowStart(recordAfterHandoff: false)
|
|
traceState("keyboardOpen.autoArmPiP")
|
|
}
|
|
|
|
func refreshSessionState() {
|
|
FlowSessionBridge.reloadFromDisk()
|
|
refreshConfigFromAppGroup()
|
|
refreshFlowPartialIfNeeded()
|
|
discardPendingUnsupportedLegacyDeliveryIfNeeded()
|
|
adoptPendingResultIfNeeded()
|
|
consumePendingFlowDeliveryIfNeeded()
|
|
consumeEditStartFailureIfNeeded()
|
|
consumeAIStartFailureIfNeeded()
|
|
|
|
recoverFromDeadHostIfNeeded()
|
|
|
|
if FlowSessionBridge.clearIfHostStale() {
|
|
debug("cleared zombie Flow session from App Group")
|
|
}
|
|
|
|
// A stale "session ended" hint may linger from an earlier drop. If the
|
|
// host is provably ready again, recover to idle now so the mic can go
|
|
// green immediately instead of waiting out the auto-clear timer.
|
|
if case .error(.flowSessionExpired, _) = state.phase,
|
|
FlowSessionBridge.isHostReady() {
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
}
|
|
|
|
recomputeMicVoiceAvailability()
|
|
promoteEditRecordingFromSnapshotIfNeeded()
|
|
startHostReadyWaitIfNeeded()
|
|
// Proactive host auto-launch is disabled (FlowHandoffPolicy): a single
|
|
// stale ready snapshot after finalize must never open startflow.
|
|
|
|
// Only surface "session ended" when the session contract *genuinely*
|
|
// dropped (expired / cleared). A transient host-ready flap — engine
|
|
// hiccup or a stale cross-process read while the session is still
|
|
// valid — must never nuke a healthy ready state into a sticky error,
|
|
// otherwise the error phase forces the mic orange and defeats the
|
|
// ready-wait poll until the auto-clear fires.
|
|
let sessionActive = FlowSessionBridge.isSessionActive()
|
|
if wasSessionActive && !sessionActive && !isFlowRecording && !isPendingFlowStart {
|
|
switch state.phase {
|
|
case .recording, .processing:
|
|
break
|
|
default:
|
|
showFlowSessionExpiredHint()
|
|
}
|
|
}
|
|
wasSessionActive = sessionActive
|
|
}
|
|
|
|
private func promoteEditRecordingFromSnapshotIfNeeded() {
|
|
guard currentUtteranceRequest?.isEdit == true, !editHostConfirmed else { return }
|
|
guard let snapshot = FlowSessionBridge.readySnapshot(),
|
|
snapshot.reason == .recording,
|
|
snapshot.busyUtteranceId == currentUtteranceId else {
|
|
return
|
|
}
|
|
editHostConfirmed = true
|
|
state.phase = .recording
|
|
startUtteranceCountdown()
|
|
onEditHostRecordingConfirmed()
|
|
}
|
|
|
|
private func consumeEditStartFailureIfNeeded() {
|
|
guard currentUtteranceRequest?.isEdit == true,
|
|
let result = matchingResult(),
|
|
isTerminalFailure(result) else {
|
|
return
|
|
}
|
|
onEditFailure(
|
|
result.text ?? ExtL10n.string("keyboard.edit.error.startTimeout")
|
|
)
|
|
completeEditResult(result, outcome: .rejected)
|
|
}
|
|
|
|
private func consumeAIStartFailureIfNeeded() {
|
|
guard currentUtteranceRequest?.isAIQuestion == true,
|
|
let result = matchingResult(),
|
|
isTerminalFailure(result) else {
|
|
return
|
|
}
|
|
completeAIFailure(result)
|
|
}
|
|
|
|
private func recomputeMicVoiceAvailability() {
|
|
FlowSessionBridge.reloadFromDisk()
|
|
let readySnapshot = FlowSessionBridge.readySnapshot()
|
|
activeSessionId = readySnapshot?.sessionId ?? activeSessionId
|
|
|
|
// If the host is mid-utterance but this extension process lost local
|
|
// ownership (jetsam / recreate after app switch), re-adopt it so we
|
|
// show red/white instead of a fake orange "starting" state.
|
|
adoptHostBusyStateIfNeeded(snapshot: readySnapshot)
|
|
|
|
let hostReadyRaw = readySnapshot?.ready == true && FlowSessionBridge.isHostReady()
|
|
let sessionActive = FlowSessionBridge.isSessionActive()
|
|
let now = Date().timeIntervalSince1970
|
|
if hostReadyRaw {
|
|
lastHostReadyAt = now
|
|
sessionProvenReady = true
|
|
}
|
|
if !sessionActive {
|
|
sessionProvenReady = false
|
|
lastHostReadyAt = 0
|
|
}
|
|
// Grace window: the host was ready very recently, so treat a momentary
|
|
// stale heartbeat read as "still warming" rather than an outright
|
|
// failure. `isSessionActive` is heartbeat-independent, so it stays true
|
|
// across cross-process read jitter and anchors this smoothing.
|
|
let withinReadyGrace = lastHostReadyAt > 0
|
|
&& (now - lastHostReadyAt) <= Self.hostReadyGrace
|
|
// Host busy (recording/processing) is NOT "still starting". Treating
|
|
// it as preparingSession was the orange-stuck bug after cold start:
|
|
// host utt.rec=1 → ready=false → keyboard forever "正在启动…".
|
|
let hostBusy = FlowKeyboardHostWarming.isHostBusy(reason: readySnapshot?.reason)
|
|
// Hold green after the session already proved ready — PiP mic release /
|
|
// ack lag must not flash yellow「正在启动…」.
|
|
let holdReady = FlowKeyboardHostWarming.shouldHoldReady(
|
|
hostReady: hostReadyRaw,
|
|
hostBusy: hostBusy,
|
|
sessionActive: sessionActive,
|
|
sessionProvenReady: sessionProvenReady,
|
|
isPendingFlowStart: isPendingFlowStart,
|
|
snapshotReason: readySnapshot?.reason
|
|
)
|
|
let hostReady = hostReadyRaw || holdReady
|
|
// PiP sessions publish `reason=.starting` while the small window is
|
|
// coming up — treat that as warming so the mic stays orange (wait)
|
|
// instead of jumping into another cold start.
|
|
let hostWarming = FlowKeyboardHostWarming.isHostWarming(
|
|
hostReady: hostReady,
|
|
hostBusy: hostBusy,
|
|
sessionActive: sessionActive,
|
|
hostReachable: FlowSessionBridge.isHostReachable(),
|
|
isPendingFlowStart: isPendingFlowStart,
|
|
withinReadyGrace: withinReadyGrace,
|
|
snapshotReason: readySnapshot?.reason
|
|
)
|
|
state.flowSessionActive = sessionActive
|
|
state.micVoiceAvailability = MicVoiceAvailabilityResolver.resolve(
|
|
phase: state.phase,
|
|
micDisabled: state.micDisabled,
|
|
hasFullAccess: hasFullAccess(),
|
|
appGroupAvailable: AppGroup.isAvailable,
|
|
hostReady: hostReady,
|
|
isPreparingSession: isPendingFlowStart || hostWarming,
|
|
hasCompletedOnboarding: voiceSetupReady
|
|
)
|
|
let signature = [
|
|
"phase=\(String(describing: state.phase))",
|
|
"availability=\(String(describing: state.micVoiceAvailability))",
|
|
hostReady ? "hostReady=1" : "hostReady=0",
|
|
state.flowSessionActive ? "sessionActive=1" : "sessionActive=0",
|
|
isPendingFlowStart ? "pending=1" : "pending=0",
|
|
isFlowRecording ? "recording=1" : "recording=0",
|
|
isAwaitingFlowResult ? "awaiting=1" : "awaiting=0",
|
|
readySnapshot?.reason.rawValue ?? "snapshot=nil"
|
|
].joined(separator: "|")
|
|
if signature != lastAvailabilityTraceSignature {
|
|
lastAvailabilityTraceSignature = signature
|
|
traceState("availability.update", extra: signature)
|
|
}
|
|
}
|
|
|
|
/// Re-attach to a host utterance this keyboard process no longer owns.
|
|
private func adoptHostBusyStateIfNeeded(snapshot: FlowReadySnapshot?) {
|
|
if currentUtteranceRequest?.isEdit != true,
|
|
let snapshot,
|
|
let busyID = snapshot.busyUtteranceId,
|
|
!cancelledEditUtteranceIDs.contains(busyID),
|
|
let command = FlowSessionBridge.latestCommand(),
|
|
command.utteranceId == busyID,
|
|
command.resolvedUtteranceMode == .editLastInput {
|
|
let orphanAction = FlowKeyboardAdoptBusyPolicy.decide(
|
|
snapshot: snapshot,
|
|
currentHostGeneration: FlowSessionBridge.currentHostGeneration(),
|
|
isFlowRecording: isFlowRecording,
|
|
isAwaitingFlowResult: isAwaitingFlowResult,
|
|
lastConsumedUtteranceId: lastConsumedUtteranceId,
|
|
lastStoppedUtteranceId: lastStoppedUtteranceId
|
|
)
|
|
guard case .adoptRecording(let sessionID, _) = orphanAction else {
|
|
return
|
|
}
|
|
activeSessionId = sessionID
|
|
currentUtteranceId = busyID
|
|
cancelledEditUtteranceIDs.insert(busyID)
|
|
writeCommand(.abort)
|
|
isFlowRecording = false
|
|
isAwaitingFlowResult = true
|
|
startFlowResultWatchdog(purpose: .cancellation)
|
|
traceState(
|
|
"orphanedEdit.failClosed",
|
|
extra: "utterance=\(busyID.uuidString.prefix(8))"
|
|
)
|
|
return
|
|
}
|
|
let action = FlowKeyboardAdoptBusyPolicy.decide(
|
|
snapshot: snapshot,
|
|
currentHostGeneration: FlowSessionBridge.currentHostGeneration(),
|
|
isFlowRecording: isFlowRecording,
|
|
isAwaitingFlowResult: isAwaitingFlowResult,
|
|
lastConsumedUtteranceId: lastConsumedUtteranceId,
|
|
lastStoppedUtteranceId: lastStoppedUtteranceId
|
|
)
|
|
switch action {
|
|
case .none:
|
|
return
|
|
case .clearStickyProcessing:
|
|
clearStickyProcessingIfNeeded(hostReady: snapshot?.ready ?? false)
|
|
case .adoptRecording(let sessionId, let busyId):
|
|
// Require the host's utterance id — inventing one makes matchingResult
|
|
// forever miss the real delivery and leaves the mic white forever.
|
|
activeSessionId = sessionId
|
|
currentUtteranceId = busyId
|
|
isPendingFlowStart = false
|
|
flowStartDeadline = 0
|
|
stopHostReadyWait()
|
|
isFlowRecording = true
|
|
state.phase = .recording
|
|
if state.lastTranscript.isEmpty {
|
|
state.lastTranscript = ""
|
|
}
|
|
if let view = wakeLockView() {
|
|
ExtensionScreenWakeLock.acquire(from: view)
|
|
}
|
|
startUtteranceCountdown()
|
|
startFlowLevelWatchdog()
|
|
traceState("adoptHostBusy.recording", extra: "session=\(sessionId)")
|
|
case .adoptProcessing(let sessionId, let busyId):
|
|
activeSessionId = sessionId
|
|
currentUtteranceId = busyId
|
|
isPendingFlowStart = false
|
|
flowStartDeadline = 0
|
|
isFlowRecording = false
|
|
stopUtteranceCountdown()
|
|
ExtensionScreenWakeLock.release()
|
|
if FlowKeyboardAdoptBusyPolicy.isStaleDeliveredProcessing(
|
|
busyUtteranceId: busyId,
|
|
latestResult: FlowSessionBridge.latestResult(),
|
|
latestAck: FlowSessionBridge.latestAck()
|
|
) {
|
|
// Already acked, result gone, host forgot to drop the gate.
|
|
// Abort unsticks the host. Do not await — claimTerminal already
|
|
// ran, so abort will not write a new result. Remember the id
|
|
// so this same refresh cannot re-adopt before the host poll.
|
|
writeCommand(.abort)
|
|
lastConsumedUtteranceId = busyId
|
|
lastStoppedUtteranceId = busyId
|
|
isAwaitingFlowResult = false
|
|
currentUtteranceId = nil
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
traceState(
|
|
"adoptHostBusy.staleProcessingReleased",
|
|
extra: "utterance=\(busyId.uuidString.prefix(8))"
|
|
)
|
|
return
|
|
}
|
|
// Missing result with no matching ack is live ASR/LLM — wait.
|
|
state.phase = .processing
|
|
if state.lastTranscript.isEmpty {
|
|
state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
|
|
}
|
|
startFlowResultWatchdog()
|
|
traceState("adoptHostBusy.processing", extra: "session=\(sessionId)")
|
|
}
|
|
}
|
|
|
|
/// After insert, a stale `reason=processing` snapshot can bounce the mic
|
|
/// back to white loading. When the host is no longer busy, force idle.
|
|
private func clearStickyProcessingIfNeeded(hostReady: Bool) {
|
|
guard !isAwaitingFlowResult, !isFlowRecording else { return }
|
|
// Edit review intentionally keeps the terminal result unacknowledged
|
|
// until the user confirms or closes. Clearing its identity here makes
|
|
// the same result get re-adopted and re-haptic on every refresh.
|
|
guard !state.editSession.isActive else { return }
|
|
guard case .processing = state.phase else { return }
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
stopFlowWatchdog()
|
|
currentUtteranceId = nil
|
|
lastStoppedUtteranceId = nil
|
|
traceState(
|
|
"stickyProcessing.cleared",
|
|
extra: hostReady ? "hostReady=1" : "hostReady=0"
|
|
)
|
|
}
|
|
|
|
/// Session is live but the ready contract has not landed yet — poll
|
|
/// quickly instead of sticking on "session inactive" orange.
|
|
///
|
|
/// Cold-start (`osgkeyboard://startflow`) is allowed only when the user
|
|
/// explicitly pressed the mic (`recordWhenHostReady`). An idle open must
|
|
/// never relaunch the host: Flow + ASR warmup then jetsams the keyboard.
|
|
private func startHostReadyWaitIfNeeded() {
|
|
guard !isPendingFlowStart else { return }
|
|
guard FlowSessionBridge.isSessionActive() else {
|
|
stopHostReadyWait()
|
|
if recordWhenHostReady {
|
|
// Session gone while waiting — escalate to a real cold start.
|
|
let shouldRecord = recordWhenHostReady
|
|
recordWhenHostReady = false
|
|
beginFlowStart(recordAfterHandoff: shouldRecord)
|
|
}
|
|
return
|
|
}
|
|
// Host busy ≠ waiting for ready. Do not spin the ready-wait poll.
|
|
if let reason = FlowSessionBridge.readySnapshot()?.reason,
|
|
reason == .recording || reason == .processing {
|
|
stopHostReadyWait()
|
|
recordWhenHostReady = false
|
|
return
|
|
}
|
|
if FlowSessionBridge.isHostReady() {
|
|
stopHostReadyWait()
|
|
finishHostReadyWaitIfNeeded()
|
|
return
|
|
}
|
|
|
|
// No mic intent + host already dead → leave cleanup to clearIfHostStale.
|
|
// Starting a wait poll here previously ended in an unprompted startflow.
|
|
if !recordWhenHostReady, isHostTrulyDeadForColdStart() {
|
|
stopHostReadyWait()
|
|
return
|
|
}
|
|
|
|
guard hostReadyWaitTask == nil else { return }
|
|
hostReadyWaitTask = Task { @MainActor [weak self] in
|
|
defer { self?.hostReadyWaitTask = nil }
|
|
for _ in 0..<20 {
|
|
guard let self, !Task.isCancelled else { return }
|
|
FlowSessionBridge.reloadFromDisk()
|
|
self.recomputeMicVoiceAvailability()
|
|
if self.state.micVoiceAvailability.isReady {
|
|
self.finishHostReadyWaitIfNeeded()
|
|
return
|
|
}
|
|
if self.state.micVoiceAvailability == .recording
|
|
|| self.state.micVoiceAvailability == .processing {
|
|
self.recordWhenHostReady = false
|
|
return
|
|
}
|
|
// Host died mid-wait — cold-start only after debounced dead
|
|
// samples AND an explicit mic-driven record intent.
|
|
let dead = self.isHostTrulyDeadForColdStart()
|
|
if self.coldStartDebouncer.observe(hostTrulyDead: dead) {
|
|
let shouldRecord = self.recordWhenHostReady
|
|
self.recordWhenHostReady = false
|
|
self.coldStartDebouncer.reset()
|
|
if shouldRecord {
|
|
self.beginFlowStart(recordAfterHandoff: true)
|
|
} else {
|
|
self.traceState(
|
|
"hostReadyWait.deadWithoutIntent",
|
|
extra: "skipColdStart=1"
|
|
)
|
|
self.stopHostReadyWait()
|
|
}
|
|
return
|
|
}
|
|
try? await Task.sleep(nanoseconds: 150_000_000)
|
|
}
|
|
// Timed out still not ready — if the user asked to record, cold-start.
|
|
guard let self else { return }
|
|
if self.recordWhenHostReady {
|
|
let shouldRecord = self.recordWhenHostReady
|
|
self.recordWhenHostReady = false
|
|
self.beginFlowStart(recordAfterHandoff: shouldRecord)
|
|
}
|
|
}
|
|
}
|
|
|
|
private func isHostTrulyDeadForColdStart() -> Bool {
|
|
FlowHandoffPolicy.shouldOpenHostColdStart(
|
|
sessionActive: FlowSessionBridge.isSessionActive(),
|
|
hostReachable: FlowSessionBridge.isHostReachable(),
|
|
hostStale: FlowSessionBridge.isHostStale(),
|
|
withinReadyGrace: false
|
|
)
|
|
}
|
|
|
|
private func finishHostReadyWaitIfNeeded() {
|
|
coldStartDebouncer.reset()
|
|
guard recordWhenHostReady else { return }
|
|
recordWhenHostReady = false
|
|
guard state.micVoiceAvailability.isReady else { return }
|
|
startFlowRecording()
|
|
traceState("hostReadyWait.recordStarted")
|
|
}
|
|
|
|
private func stopHostReadyWait() {
|
|
hostReadyWaitTask?.cancel()
|
|
hostReadyWaitTask = nil
|
|
}
|
|
|
|
func toggleRecording() {
|
|
switch state.phase {
|
|
case .recording:
|
|
pressEnded()
|
|
case .requestingPermissions:
|
|
break
|
|
case .idle, .denied, .error:
|
|
// A second tap while still waiting for the host must cancel the
|
|
// pending start instead of stacking another utterance.
|
|
if currentUtteranceRequest != nil {
|
|
cancelCurrentDictation()
|
|
return
|
|
}
|
|
_ = startUtterance(.dictation)
|
|
case .processing:
|
|
break
|
|
}
|
|
}
|
|
|
|
func setMicTouchActive(_ active: Bool) {
|
|
if active {
|
|
beginAudioPrimeIfPossible()
|
|
} else if currentUtteranceRequest != nil {
|
|
audioPrimeCancelTask?.cancel()
|
|
audioPrimeCancelTask = nil
|
|
audioPrimeID = nil
|
|
}
|
|
// Do not synchronously cancel on finger-up. SwiftUI may resolve the
|
|
// tap/hold action after this callback; the bounded timer handles a
|
|
// touch that is never adopted by an utterance.
|
|
}
|
|
|
|
private func beginAudioPrimeIfPossible() {
|
|
guard currentUtteranceRequest == nil,
|
|
currentUtteranceId == nil,
|
|
!isPendingFlowStart,
|
|
!isAwaitingFlowResult,
|
|
state.phase == .idle,
|
|
FlowSessionBridge.isHostReady(),
|
|
let sessionID = FlowSessionBridge.readySnapshot()?.sessionId,
|
|
audioPrimeID == nil else {
|
|
return
|
|
}
|
|
let primeID = UUID()
|
|
audioPrimeID = primeID
|
|
let command = FlowCommand(
|
|
sessionId: sessionID,
|
|
utteranceId: primeID,
|
|
commandSeq: nextCommandSeq(),
|
|
action: .primeAudio,
|
|
localeId: state.localeId
|
|
)
|
|
FlowSessionBridge.writeCommand(command)
|
|
audioPrimeCancelTask?.cancel()
|
|
audioPrimeCancelTask = Task { @MainActor [weak self] in
|
|
try? await Task.sleep(nanoseconds: 4_000_000_000)
|
|
guard !Task.isCancelled else { return }
|
|
self?.cancelAudioPrimeIfNeeded()
|
|
}
|
|
}
|
|
|
|
private func cancelAudioPrimeIfNeeded() {
|
|
audioPrimeCancelTask?.cancel()
|
|
audioPrimeCancelTask = nil
|
|
guard let primeID = audioPrimeID,
|
|
let sessionID = FlowSessionBridge.readySnapshot()?.sessionId else {
|
|
audioPrimeID = nil
|
|
return
|
|
}
|
|
audioPrimeID = nil
|
|
FlowSessionBridge.writeCommand(
|
|
FlowCommand(
|
|
sessionId: sessionID,
|
|
utteranceId: primeID,
|
|
commandSeq: nextCommandSeq(),
|
|
action: .cancelPrimeAudio,
|
|
localeId: state.localeId
|
|
)
|
|
)
|
|
}
|
|
|
|
func beginEditRecording(
|
|
reference: EditableInputReference
|
|
) -> FlowUtteranceStartDisposition {
|
|
startUtterance(.editLastInput(reference))
|
|
}
|
|
|
|
func beginAIRecording(
|
|
conversationID: UUID,
|
|
oobeFeature: ManagedGatewayOOBEFeature? = nil
|
|
) -> FlowUtteranceStartDisposition {
|
|
startUtterance(
|
|
.aiQuestion(
|
|
conversationID: conversationID,
|
|
oobeFeature: oobeFeature
|
|
)
|
|
)
|
|
}
|
|
|
|
func submitAIQuestion(
|
|
text: String,
|
|
conversationID: UUID,
|
|
taskKind: ManagedGatewayTaskKind = .aiQuestion,
|
|
requestSource: ManagedGatewayRequestSource? = nil,
|
|
oobeFeature: ManagedGatewayOOBEFeature? = nil,
|
|
thinkingEnabled: Bool? = nil,
|
|
webPageURL: URL? = nil
|
|
) -> FlowUtteranceStartDisposition {
|
|
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
guard !trimmed.isEmpty else { return .rejected(.pipelineBusy) }
|
|
return startUtterance(
|
|
.aiQuestion(
|
|
conversationID: conversationID,
|
|
prefilledQuestion: trimmed,
|
|
taskKind: taskKind,
|
|
requestSource: requestSource,
|
|
oobeFeature: oobeFeature,
|
|
thinkingEnabled: thinkingEnabled,
|
|
webPageURL: webPageURL
|
|
)
|
|
)
|
|
}
|
|
|
|
func stopAIRecording() {
|
|
guard currentUtteranceRequest?.isAIQuestion == true else { return }
|
|
pressEnded()
|
|
}
|
|
|
|
func cancelAIRecording() {
|
|
guard currentUtteranceRequest?.isAIQuestion == true else { return }
|
|
prepareLocalCancel()
|
|
beginAwaitingAbort(kind: .aiQuestion)
|
|
}
|
|
|
|
func endAIConversation(_ conversationID: UUID) {
|
|
guard let sessionID = FlowSessionBridge.readySnapshot()?.sessionId
|
|
?? activeSessionId else {
|
|
return
|
|
}
|
|
FlowSessionBridge.writeCommand(
|
|
FlowCommand(
|
|
sessionId: sessionID,
|
|
utteranceId: UUID(),
|
|
commandSeq: nextCommandSeq(),
|
|
action: .endAIConversation,
|
|
localeId: state.localeId,
|
|
aiConversationID: conversationID
|
|
)
|
|
)
|
|
}
|
|
|
|
func stopEditRecording() {
|
|
guard currentUtteranceRequest?.isEdit == true else { return }
|
|
pressEnded()
|
|
}
|
|
|
|
func cancelCurrentDictation() {
|
|
if currentUtteranceRequest?.isAIQuestion == true {
|
|
cancelAIRecording()
|
|
return
|
|
}
|
|
guard currentUtteranceRequest?.isEdit != true else { return }
|
|
|
|
prepareLocalCancel()
|
|
let hadIssuedTransport = currentUtteranceId != nil
|
|
&& (isFlowRecording || isAwaitingFlowResult)
|
|
beginAwaitingAbort(kind: .dictation)
|
|
traceState(
|
|
"dictation.cancelled",
|
|
extra: hadIssuedTransport
|
|
? "utterance=\(currentUtteranceId?.uuidString.prefix(8) ?? "none")"
|
|
: "transport=localOnly"
|
|
)
|
|
}
|
|
|
|
/// Stop local wait/prime timers before an abort. Does not change phase;
|
|
/// `beginAwaitingAbort` keeps `.processing` until the host acks.
|
|
private func prepareLocalCancel() {
|
|
recordWhenHostReady = false
|
|
recordAfterHandoff = false
|
|
isPendingFlowStart = false
|
|
flowStartDeadline = 0
|
|
coldStartDebouncer.reset()
|
|
stopHostReadyWait()
|
|
stopFlowWatchdog()
|
|
stopUtteranceCountdown()
|
|
ExtensionScreenWakeLock.release()
|
|
state.level = 0
|
|
state.lastTranscript = ""
|
|
}
|
|
|
|
/// Keep the cancel chrome (X + white mic) until the host finishes abort.
|
|
private func beginAwaitingAbort(kind: VoiceCancellationKind) {
|
|
guard let utteranceID = currentUtteranceId,
|
|
isFlowRecording || isAwaitingFlowResult else {
|
|
finishLocalCancel()
|
|
return
|
|
}
|
|
let inserted = registerCancelledUtterance(utteranceID, kind: kind)
|
|
if !inserted, isAwaitingFlowResult {
|
|
state.phase = .processing
|
|
recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
writeCommand(.abort)
|
|
isFlowRecording = false
|
|
isAwaitingFlowResult = true
|
|
state.phase = .processing
|
|
startFlowResultWatchdog(purpose: .cancellation)
|
|
recomputeMicVoiceAvailability()
|
|
}
|
|
|
|
/// Register cancellation before starting the watchdog. Keeping the mutation
|
|
/// in this leaf method prevents a stored-property `inout` access from
|
|
/// overlapping watchdog reads and triggering Swift's exclusivity trap.
|
|
private func registerCancelledUtterance(
|
|
_ utteranceID: UUID,
|
|
kind: VoiceCancellationKind
|
|
) -> Bool {
|
|
switch kind {
|
|
case .dictation:
|
|
return cancelledDictationUtteranceIDs.insert(utteranceID).inserted
|
|
case .aiQuestion:
|
|
return cancelledAIUtteranceIDs.insert(utteranceID).inserted
|
|
}
|
|
}
|
|
|
|
private func finishLocalCancel() {
|
|
clearUnissuedUtterance()
|
|
isFlowRecording = false
|
|
isAwaitingFlowResult = false
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
}
|
|
|
|
func abortEditRecording() {
|
|
guard currentUtteranceRequest?.isEdit == true else { return }
|
|
recordWhenHostReady = false
|
|
recordAfterHandoff = false
|
|
isPendingFlowStart = false
|
|
flowStartDeadline = 0
|
|
coldStartDebouncer.reset()
|
|
stopHostReadyWait()
|
|
stopFlowWatchdog()
|
|
if let currentUtteranceId {
|
|
cancelledEditUtteranceIDs.insert(currentUtteranceId)
|
|
writeCommand(.abort)
|
|
isAwaitingFlowResult = true
|
|
isFlowRecording = false
|
|
currentUtteranceRequest = nil
|
|
editHostConfirmed = false
|
|
stopUtteranceCountdown()
|
|
ExtensionScreenWakeLock.release()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
startFlowResultWatchdog(purpose: .cancellation)
|
|
return
|
|
}
|
|
resetEditTransportState()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
}
|
|
|
|
func acknowledgeEditResult(_ outcome: FlowAck.DeliveryOutcome) {
|
|
FlowSessionBridge.reloadFromDisk()
|
|
guard let result = matchingResult(),
|
|
result.resolvedUtteranceMode == .editLastInput else {
|
|
// Review can only exist after a terminal edit result was received.
|
|
// If App Group cleanup won the race, finish locally instead of
|
|
// aborting an already-completed utterance and blocking the next edit.
|
|
if let currentUtteranceId {
|
|
lastConsumedUtteranceId = currentUtteranceId
|
|
cancelledEditUtteranceIDs.remove(currentUtteranceId)
|
|
}
|
|
lastStoppedUtteranceId = nil
|
|
resetEditTransportState()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
completeEditResult(result, outcome: outcome)
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
}
|
|
|
|
private func completeEditResult(
|
|
_ result: FlowResult,
|
|
outcome: FlowAck.DeliveryOutcome
|
|
) {
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision,
|
|
deliveryOutcome: outcome
|
|
)
|
|
)
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
lastStoppedUtteranceId = nil
|
|
cancelledEditUtteranceIDs.remove(result.utteranceId)
|
|
resetEditTransportState()
|
|
}
|
|
|
|
private func completeAIResult(_ result: FlowResult) {
|
|
onAIResult(result)
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision
|
|
)
|
|
)
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
lastStoppedUtteranceId = nil
|
|
resetEditTransportState()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
}
|
|
|
|
private func completeAIFailure(_ result: FlowResult) {
|
|
onAIFailure(
|
|
result.text ?? ExtL10n.string("keyboard.ai.error.requestFailed"),
|
|
result.utteranceId
|
|
)
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision,
|
|
deliveryOutcome: .rejected
|
|
)
|
|
)
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
lastStoppedUtteranceId = nil
|
|
resetEditTransportState()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
}
|
|
|
|
func pressBegan() {
|
|
_ = startUtterance(.dictation)
|
|
}
|
|
|
|
@discardableResult
|
|
private func startUtterance(
|
|
_ request: FlowUtteranceRequest
|
|
) -> FlowUtteranceStartDisposition {
|
|
switch state.phase {
|
|
case .idle, .denied, .error:
|
|
break
|
|
default:
|
|
return .rejected(.pipelineBusy)
|
|
}
|
|
if let currentUtteranceId, currentUtteranceRequest != nil {
|
|
return .alreadyInFlight(currentUtteranceId)
|
|
}
|
|
guard cancelledEditUtteranceIDs.isEmpty, !isAwaitingFlowResult else {
|
|
return .rejected(.pipelineBusy)
|
|
}
|
|
let utteranceID = UUID()
|
|
// The common utterance transaction adopts any capture primed by this
|
|
// same touch; host-side capture startup is single-flight.
|
|
audioPrimeCancelTask?.cancel()
|
|
audioPrimeCancelTask = nil
|
|
audioPrimeID = nil
|
|
currentUtteranceId = utteranceID
|
|
currentUtteranceRequest = request
|
|
editHostConfirmed = false
|
|
if request.isAIQuestion {
|
|
onAIUtterancePrepared(utteranceID)
|
|
}
|
|
currentStartDeadlineAt = Date().timeIntervalSince1970
|
|
+ FlowSessionKeys.utteranceStartBudget
|
|
|
|
// A warm-only PiP handoff may already be in progress. Upgrade that
|
|
// same shared transaction instead of creating an edit-only wait path.
|
|
if isPendingFlowStart {
|
|
recordAfterHandoff = true
|
|
return .waitingForHost(utteranceID)
|
|
}
|
|
|
|
recomputeMicVoiceAvailability()
|
|
|
|
switch state.micVoiceAvailability {
|
|
case .unavailable(.onboardingIncomplete):
|
|
promptFinishSetupInApp()
|
|
clearUnissuedUtterance()
|
|
return .rejected(.onboardingIncomplete)
|
|
case .unavailable(.missingAPIKey):
|
|
clearUnissuedUtterance()
|
|
return .rejected(.missingAPIKey)
|
|
case .unavailable(.noFullAccess):
|
|
persistStartupFailure("noFullAccess")
|
|
let msg = ExtL10n.string("keyboard.error.fullAccessRequired")
|
|
state.phase = .error(.fullAccessRequired, message: msg)
|
|
scheduleAutoClearError()
|
|
recomputeMicVoiceAvailability()
|
|
clearUnissuedUtterance()
|
|
return .rejected(.noFullAccess)
|
|
case .unavailable(.appGroupUnavailable):
|
|
// Falls back to this extension's own container — the App Group is
|
|
// exactly what is missing here — so it will not show in Settings.
|
|
persistStartupFailure("appGroupUnavailable")
|
|
let msg = ExtL10n.string("keyboard.error.appGroupCommunication")
|
|
state.phase = .error(.appGroupUnavailable, message: msg)
|
|
scheduleAutoClearError()
|
|
recomputeMicVoiceAvailability()
|
|
clearUnissuedUtterance()
|
|
return .rejected(.appGroupUnavailable)
|
|
default:
|
|
break
|
|
}
|
|
|
|
let withinReadyGrace = lastHostReadyAt > 0
|
|
&& (Date().timeIntervalSince1970 - lastHostReadyAt) <= Self.hostReadyGrace
|
|
let action = FlowHandoffPolicy.micPressAction(
|
|
availability: state.micVoiceAvailability,
|
|
sessionActive: FlowSessionBridge.isSessionActive(),
|
|
hostReachable: FlowSessionBridge.isHostReachable(),
|
|
hostStale: FlowSessionBridge.isHostStale(),
|
|
withinReadyGrace: withinReadyGrace
|
|
)
|
|
switch action {
|
|
case .startRecording:
|
|
detectAndStoreAppContext()
|
|
startFlowRecording()
|
|
if FlowSessionBridge.latestCommand()?.utteranceId == utteranceID {
|
|
return .issued(utteranceID)
|
|
}
|
|
if isPendingFlowStart || recordWhenHostReady {
|
|
return .waitingForHost(utteranceID)
|
|
}
|
|
// Availability said ready, yet no command was issued and no wait is
|
|
// pending: the press is dropped with no error UI at all.
|
|
persistStartupFailure("pressDroppedHostUnavailable")
|
|
clearUnissuedUtterance()
|
|
return .rejected(.hostUnavailable)
|
|
case .waitForHostReady(let recordWhenReady):
|
|
detectAndStoreAppContext()
|
|
recordWhenHostReady = recordWhenReady
|
|
coldStartDebouncer.reset()
|
|
startHostReadyWaitIfNeeded()
|
|
traceState(
|
|
"pressBegan.waitForHostReady",
|
|
extra: recordWhenReady ? "recordWhenReady=1" : "recordWhenReady=0"
|
|
)
|
|
return .waitingForHost(utteranceID)
|
|
case .openHostColdStart:
|
|
detectAndStoreAppContext()
|
|
beginFlowStart(recordAfterHandoff: true)
|
|
return .waitingForHost(utteranceID)
|
|
case .ignore:
|
|
clearUnissuedUtterance()
|
|
return .rejected(.pipelineBusy)
|
|
}
|
|
}
|
|
|
|
private func clearUnissuedUtterance() {
|
|
currentUtteranceId = nil
|
|
currentUtteranceRequest = nil
|
|
currentStartDeadlineAt = nil
|
|
recordAfterHandoff = false
|
|
recordWhenHostReady = false
|
|
}
|
|
|
|
func pressEnded() {
|
|
if isPendingFlowStart {
|
|
cancelPendingFlowStart()
|
|
return
|
|
}
|
|
guard isFlowRecording else { return }
|
|
|
|
isFlowRecording = false
|
|
stopUtteranceCountdown()
|
|
ExtensionScreenWakeLock.release()
|
|
lastStoppedUtteranceId = currentUtteranceId
|
|
writeCommand(.stopRecording)
|
|
debug("pressEnded wrote stop command")
|
|
state.phase = .processing
|
|
if currentUtteranceRequest?.isAIQuestion == true,
|
|
let currentUtteranceId {
|
|
onAIRecognitionStarted(currentUtteranceId)
|
|
} else if currentUtteranceRequest?.isEdit != true {
|
|
state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
|
|
}
|
|
startFlowResultWatchdog()
|
|
recomputeMicVoiceAvailability()
|
|
}
|
|
|
|
func beginFlowStart(recordAfterHandoff: Bool = false) {
|
|
guard voiceSetupReady else {
|
|
persistStartupFailure("voiceSetupIncomplete")
|
|
promptFinishSetupInApp()
|
|
return
|
|
}
|
|
guard !isPendingFlowStart else {
|
|
traceState("beginFlowStart.ignored", extra: "reason=pendingAlreadyTrue")
|
|
return
|
|
}
|
|
self.recordAfterHandoff = recordAfterHandoff
|
|
recordWhenHostReady = false
|
|
coldStartDebouncer.reset()
|
|
isPendingFlowStart = true
|
|
isFlowRecording = false
|
|
coldStartForegroundHintShown = false
|
|
let now = Date().timeIntervalSince1970
|
|
// This path always jumps to the host (`openHostApp("startflow")`), so
|
|
// the budget must cover host foreground + PiP arm + first ready publish.
|
|
// Anchor it at the jump (not the mic press) with the cold-launch budget;
|
|
// reusing the press-anchored 8 s charged app-launch latency against the
|
|
// recording budget and timed out legitimate cold starts.
|
|
flowStartDeadline = now + FlowSessionKeys.coldStartHostLaunchBudget
|
|
flowStartHardDeadline = now + FlowSessionKeys.coldStartHostLaunchHardCap
|
|
currentStartDeadlineAt = flowStartDeadline
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
FlowSessionBridge.markPiPArmAttempt()
|
|
OSGDiag.log(
|
|
"beginFlowStart → openHostApp(startflow) recordAfterHandoff=\(recordAfterHandoff) "
|
|
+ "\(OSGDiag.memoryTag())",
|
|
category: "boot"
|
|
)
|
|
openHostApp("startflow")
|
|
startFlowStartWatchdog()
|
|
traceState(
|
|
"beginFlowStart.started",
|
|
extra: recordAfterHandoff ? "recordAfterHandoff=1" : "recordAfterHandoff=0"
|
|
)
|
|
}
|
|
|
|
func handleHostAppOpenResult(path: String, success: Bool) {
|
|
traceState("openHostApp.result", extra: "path=\(path) success=\(success)")
|
|
guard !success else { return }
|
|
|
|
// The open genuinely failed (iOS blocked it / no Full Access). Don't
|
|
// let the start watchdog spin — cancel the pending start immediately and
|
|
// guide the user to open OSGKeyboard manually.
|
|
if path == "startflow", isPendingFlowStart {
|
|
isPendingFlowStart = false
|
|
recordAfterHandoff = false
|
|
flowStartDeadline = 0
|
|
stopFlowWatchdog()
|
|
traceState("openHostApp.failed", extra: "path=startflow cancelPending=1")
|
|
// iOS refused the jump (no Full Access, or the open was blocked):
|
|
// the host process never even had a chance to record anything.
|
|
persistStartupFailure("hostOpenRejected")
|
|
if currentUtteranceRequest?.isEdit == true {
|
|
onEditFailure(ExtL10n.string("keyboard.error.manualOpenForFlow"))
|
|
resetEditTransportState()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
return
|
|
}
|
|
if currentUtteranceRequest?.isAIQuestion == true {
|
|
onAIFailure(
|
|
ExtL10n.string("keyboard.error.manualOpenForFlow"),
|
|
currentUtteranceId
|
|
)
|
|
resetEditTransportState()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
return
|
|
}
|
|
showManualOpenHint(path: "startflow")
|
|
recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
|
|
showManualOpenHint(path: path)
|
|
}
|
|
|
|
func cancelPipelineUnlessAwaitingResult() {
|
|
guard !isAwaitingFlowResult else { return }
|
|
if isFlowRecording || isPendingFlowStart {
|
|
if isFlowRecording {
|
|
writeCommand(.abort)
|
|
ExtensionScreenWakeLock.release()
|
|
// Remember the aborted id so the next keyboard open cannot
|
|
// re-adopt a lagging host snapshot as 「识别中」.
|
|
lastStoppedUtteranceId = currentUtteranceId
|
|
}
|
|
currentUtteranceId = nil
|
|
isFlowRecording = false
|
|
isPendingFlowStart = false
|
|
recordAfterHandoff = false
|
|
stopUtteranceCountdown()
|
|
stopFlowWatchdog()
|
|
state.level = 0
|
|
recomputeMicVoiceAvailability()
|
|
}
|
|
currentUtteranceRequest = nil
|
|
editHostConfirmed = false
|
|
}
|
|
|
|
// MARK: - Private
|
|
|
|
private func nextCommandSeq() -> Int64 {
|
|
let millis = Int64(Date().timeIntervalSince1970 * 1_000)
|
|
currentCommandSeq = max(currentCommandSeq + 1, millis)
|
|
return currentCommandSeq
|
|
}
|
|
|
|
private func writeCommand(_ action: FlowCommand.Action) {
|
|
guard let activeSessionId, let currentUtteranceId else { return }
|
|
let request = currentUtteranceRequest ?? .dictation
|
|
let mode: FlowUtteranceMode? = request.mode == .dictation
|
|
? nil
|
|
: request.mode
|
|
let implicitOOBEFeature: ManagedGatewayOOBEFeature? =
|
|
state.oobePracticeSession?.expectedFeature == .voiceInput
|
|
&& request.mode == .dictation
|
|
? .voiceInput
|
|
: nil
|
|
let managedOOBEFeature = request.managedOOBEFeature ?? implicitOOBEFeature
|
|
let managedRequestPurpose = request.managedRequestPurpose
|
|
?? (managedOOBEFeature == nil ? nil : .oobe)
|
|
let command = FlowCommand(
|
|
sessionId: activeSessionId,
|
|
utteranceId: currentUtteranceId,
|
|
commandSeq: nextCommandSeq(),
|
|
action: action,
|
|
localeId: state.localeId,
|
|
fieldContext: action == .stopRecording && !request.isAIQuestion
|
|
? fieldContextProvider()
|
|
: nil,
|
|
utteranceMode: mode,
|
|
editSourceText: action == .startRecording
|
|
? request.editSourceText
|
|
: nil,
|
|
sourceHistoryEntryID: request.sourceHistoryEntryID,
|
|
sourceHistoryEntryRevision: request.sourceHistoryEntryRevision,
|
|
aiConversationID: request.aiConversationID,
|
|
aiTaskKind: request.aiTaskKind,
|
|
managedRequestSource: request.managedRequestSource,
|
|
managedRequestPurpose: managedRequestPurpose,
|
|
managedOOBEFeature: managedOOBEFeature,
|
|
startDeadlineAt: action == .startRecording ? currentStartDeadlineAt : nil,
|
|
processingDeadlineAt: action == .stopRecording && request.isEdit
|
|
? Date().timeIntervalSince1970
|
|
+ FlowSessionKeys.editLastInputHostProcessingBudget
|
|
: nil
|
|
)
|
|
FlowSessionBridge.writeCommand(command)
|
|
if action == .startRecording, let currentStartDeadlineAt {
|
|
FlowSessionBridge.writeStartTransaction(
|
|
FlowStartTransaction(
|
|
sessionID: activeSessionId,
|
|
utteranceID: currentUtteranceId,
|
|
deadlineAt: currentStartDeadlineAt,
|
|
phase: .issued
|
|
)
|
|
)
|
|
}
|
|
debug(
|
|
"command \(action.rawValue) seq=\(command.commandSeq) " +
|
|
"utterance=\(currentUtteranceId.uuidString) mode=\(mode?.rawValue ?? "dictation") contextChars=" +
|
|
"\(command.fieldContext?.precedingText?.count ?? 0)/" +
|
|
"\(command.fieldContext?.followingText?.count ?? 0)"
|
|
)
|
|
// Start of one traceable utterance: everything the host logs afterwards
|
|
// belongs to this `utterance=` id until the matching keyboard.insert.
|
|
FlowTrace.keyboard(
|
|
"command.\(action.rawValue)",
|
|
"seq=\(command.commandSeq) utterance=\(currentUtteranceId.uuidString.prefix(8)) "
|
|
+ "locale=\(state.localeId) engine=\(state.engineMode) "
|
|
+ "hostReady=\(FlowSessionBridge.isHostReady() ? 1 : 0) "
|
|
+ "mode=\(mode?.rawValue ?? "dictation")"
|
|
)
|
|
}
|
|
|
|
private func consumePendingFlowDeliveryIfNeeded() {
|
|
if isAwaitingFlowResult {
|
|
if let result = matchingResult(),
|
|
discardUnsupportedLegacyResultIfNeeded(result) {
|
|
return
|
|
}
|
|
if let result = matchingResult(),
|
|
consumeCancelledDictationResultIfNeeded(result) {
|
|
return
|
|
}
|
|
if let result = matchingResult(),
|
|
consumeDiscardedEmptyResultIfNeeded(result) {
|
|
return
|
|
}
|
|
if let result = matchingResult(),
|
|
consumeCancelledEditResultIfNeeded(result) {
|
|
return
|
|
}
|
|
if let result = matchingResult(),
|
|
consumeCancelledAIResultIfNeeded(result) {
|
|
return
|
|
}
|
|
if let result = matchingResult(), result.status == .final, let text = result.text, !text.isEmpty {
|
|
isAwaitingFlowResult = false
|
|
stopFlowWatchdog()
|
|
if result.resolvedUtteranceMode == .editLastInput {
|
|
state.phase = .processing
|
|
onEditResult(result)
|
|
return
|
|
}
|
|
if result.resolvedUtteranceMode == .aiQuestion {
|
|
completeAIResult(result)
|
|
return
|
|
}
|
|
textInserter.handleFlowTranscript(
|
|
TranscriptionDelivery(
|
|
text: text,
|
|
polishWarning: result.warning,
|
|
historyEntryID: result.historyEntryID,
|
|
historyEntryRevision: result.historyEntryRevision
|
|
)
|
|
)
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision
|
|
)
|
|
)
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
lastStoppedUtteranceId = nil
|
|
currentUtteranceId = nil
|
|
FlowTrace.transcript(
|
|
"keyboard.insert",
|
|
text,
|
|
"utterance=\(result.utteranceId.uuidString.prefix(8)) "
|
|
+ "commandSeq=\(result.commandSeq) warning=\(result.warning == nil ? 0 : 1)"
|
|
)
|
|
recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
if let result = matchingResult(), isTerminalFailure(result) {
|
|
FlowTrace.warn(
|
|
"keyboard.resultFailed",
|
|
"status=\(result.status.rawValue) "
|
|
+ "kind=\(result.errorKind?.rawValue ?? "none") "
|
|
+ "utterance=\(result.utteranceId.uuidString.prefix(8)) "
|
|
+ "messageLen=\(result.text?.count ?? 0)"
|
|
)
|
|
isAwaitingFlowResult = false
|
|
stopFlowWatchdog()
|
|
if result.resolvedUtteranceMode == .editLastInput {
|
|
onEditFailure(
|
|
result.text ?? ExtL10n.string("keyboard.edit.error.processing")
|
|
)
|
|
completeEditResult(result, outcome: .rejected)
|
|
state.phase = .idle
|
|
return
|
|
}
|
|
if result.resolvedUtteranceMode == .aiQuestion {
|
|
completeAIFailure(result)
|
|
return
|
|
}
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision
|
|
)
|
|
)
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
lastStoppedUtteranceId = nil
|
|
currentUtteranceId = nil
|
|
let error = FlowTranscriptionError(
|
|
message: result.text ?? ExtL10n.string("keyboard.flow.resultTimeout"),
|
|
kind: result.errorKind ?? .generic
|
|
)
|
|
state.phase = .error(
|
|
.fromFlowTranscription(error),
|
|
message: error.message
|
|
)
|
|
scheduleAutoClearError()
|
|
recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
}
|
|
|
|
if isPendingFlowStart, FlowSessionBridge.isHostReady() {
|
|
completeFlowStartHandoff()
|
|
}
|
|
}
|
|
|
|
private func consumeCancelledDictationResultIfNeeded(_ result: FlowResult) -> Bool {
|
|
guard cancelledDictationUtteranceIDs.contains(result.utteranceId),
|
|
result.status == .final || isTerminalFailure(result) else {
|
|
return false
|
|
}
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision,
|
|
deliveryOutcome: .rejected
|
|
)
|
|
)
|
|
cancelledDictationUtteranceIDs.remove(result.utteranceId)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
lastStoppedUtteranceId = nil
|
|
stopUtteranceCountdown()
|
|
stopFlowWatchdog()
|
|
ExtensionScreenWakeLock.release()
|
|
resetEditTransportState()
|
|
state.level = 0
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
traceState(
|
|
"dictation.cancelledResultDiscarded",
|
|
extra: "utterance=\(result.utteranceId.uuidString.prefix(8))"
|
|
)
|
|
return true
|
|
}
|
|
|
|
private func consumeDiscardedEmptyResultIfNeeded(_ result: FlowResult) -> Bool {
|
|
guard result.errorKind == .discardedEmpty,
|
|
result.status == .aborted || isTerminalFailure(result) else {
|
|
return false
|
|
}
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision,
|
|
deliveryOutcome: .rejected
|
|
)
|
|
)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
lastStoppedUtteranceId = nil
|
|
stopUtteranceCountdown()
|
|
stopFlowWatchdog()
|
|
ExtensionScreenWakeLock.release()
|
|
let wasAI = currentUtteranceRequest?.isAIQuestion == true
|
|
resetEditTransportState()
|
|
state.level = 0
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
if wasAI {
|
|
onAICancelled()
|
|
}
|
|
traceState(
|
|
"utterance.discardedEmptyTap",
|
|
extra: "utterance=\(result.utteranceId.uuidString.prefix(8))"
|
|
)
|
|
return true
|
|
}
|
|
|
|
private func consumeCancelledEditResultIfNeeded(_ result: FlowResult) -> Bool {
|
|
guard cancelledEditUtteranceIDs.contains(result.utteranceId),
|
|
result.status == .final || isTerminalFailure(result) else {
|
|
return false
|
|
}
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision,
|
|
deliveryOutcome: .rejected
|
|
)
|
|
)
|
|
cancelledEditUtteranceIDs.remove(result.utteranceId)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
resetEditTransportState()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
return true
|
|
}
|
|
|
|
private func consumeCancelledAIResultIfNeeded(_ result: FlowResult) -> Bool {
|
|
guard cancelledAIUtteranceIDs.contains(result.utteranceId),
|
|
result.status == .final || isTerminalFailure(result) else {
|
|
return false
|
|
}
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision,
|
|
deliveryOutcome: .rejected
|
|
)
|
|
)
|
|
cancelledAIUtteranceIDs.remove(result.utteranceId)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
resetEditTransportState()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
return true
|
|
}
|
|
|
|
private func adoptPendingResultIfNeeded() {
|
|
guard !isAwaitingFlowResult, currentUtteranceId == nil,
|
|
let pendingId = FlowSessionBridge.pendingKeyboardUtteranceId(),
|
|
let result = FlowSessionBridge.latestResult(),
|
|
result.utteranceId == pendingId,
|
|
result.status == .final || isTerminalFailure(result) else {
|
|
return
|
|
}
|
|
if result.resolvedUtteranceMode == .editLastInput,
|
|
currentUtteranceRequest?.isEdit != true {
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision,
|
|
deliveryOutcome: .rejected
|
|
)
|
|
)
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
traceState(
|
|
"pendingEditResult.discarded",
|
|
extra: "reason=extensionRecreated"
|
|
)
|
|
return
|
|
}
|
|
let currentField = fieldContextProvider()
|
|
if let expected = result.fieldFingerprint,
|
|
let current = currentField?.deliveryFingerprint,
|
|
expected != current {
|
|
if let text = result.text,
|
|
currentField?.precedingText?.hasSuffix(text) == true {
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision
|
|
)
|
|
)
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
traceState(
|
|
"pendingResult.acknowledged",
|
|
extra: "reason=textAlreadyPresent"
|
|
)
|
|
return
|
|
}
|
|
traceState(
|
|
"pendingResult.deferred",
|
|
extra: "reason=fieldFingerprintMismatch"
|
|
)
|
|
return
|
|
}
|
|
activeSessionId = result.sessionId
|
|
currentUtteranceId = result.utteranceId
|
|
isAwaitingFlowResult = true
|
|
state.phase = .processing
|
|
traceState(
|
|
"pendingResult.adopted",
|
|
extra: "utterance=\(pendingId.uuidString.prefix(8))"
|
|
)
|
|
}
|
|
|
|
private func matchingResult() -> FlowResult? {
|
|
FlowKeyboardResultMatcher.matchingResult(
|
|
latest: FlowSessionBridge.latestResult(),
|
|
activeSessionId: activeSessionId,
|
|
currentUtteranceId: currentUtteranceId,
|
|
currentHostGeneration: FlowSessionBridge.currentHostGeneration()
|
|
)
|
|
}
|
|
|
|
private func isTerminalFailure(_ result: FlowResult) -> Bool {
|
|
FlowKeyboardResultMatcher.isTerminalFailure(result)
|
|
}
|
|
|
|
/// Clipboard-command results from older builds are terminally discarded.
|
|
/// Acknowledging them prevents the host from keeping a stale delivery alive.
|
|
private func discardPendingUnsupportedLegacyDeliveryIfNeeded() {
|
|
guard let result = FlowSessionBridge.latestResult(),
|
|
result.resolvedUtteranceMode == .unsupportedLegacy,
|
|
result.status == .final || isTerminalFailure(result) else {
|
|
return
|
|
}
|
|
discardUnsupportedLegacyResultIfNeeded(result)
|
|
}
|
|
|
|
@discardableResult
|
|
private func discardUnsupportedLegacyResultIfNeeded(_ result: FlowResult) -> Bool {
|
|
guard result.resolvedUtteranceMode == .unsupportedLegacy else { return false }
|
|
isAwaitingFlowResult = false
|
|
isFlowRecording = false
|
|
stopUtteranceCountdown()
|
|
stopFlowWatchdog()
|
|
ExtensionScreenWakeLock.release()
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision,
|
|
deliveryOutcome: .rejected
|
|
)
|
|
)
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
lastStoppedUtteranceId = nil
|
|
currentUtteranceId = nil
|
|
currentStartDeadlineAt = nil
|
|
currentUtteranceRequest = nil
|
|
editHostConfirmed = false
|
|
state.level = 0
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
traceState("legacyMode.discarded")
|
|
return true
|
|
}
|
|
|
|
/// When the host process died mid-utterance, abort local recording / waiting
|
|
/// so the user is not stuck until the long result watchdog fires.
|
|
private func recoverFromDeadHostIfNeeded() {
|
|
guard FlowSessionBridge.isHostStale() else { return }
|
|
|
|
if isFlowRecording {
|
|
if currentUtteranceRequest?.isEdit == true {
|
|
onEditFailure(ExtL10n.string("keyboard.flow.hostDisconnected"))
|
|
}
|
|
isFlowRecording = false
|
|
stopUtteranceCountdown()
|
|
ExtensionScreenWakeLock.release()
|
|
writeCommand(.abort)
|
|
currentUtteranceId = nil
|
|
lastStoppedUtteranceId = nil
|
|
stopFlowWatchdog()
|
|
state.level = 0
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
resetEditTransportState()
|
|
recomputeMicVoiceAvailability()
|
|
debug("aborted recording — host heartbeat zombie")
|
|
return
|
|
}
|
|
|
|
if isAwaitingFlowResult {
|
|
failHostDisconnected()
|
|
}
|
|
}
|
|
|
|
private func failHostDisconnected() {
|
|
// Host was alive enough to be asked, then went away (jetsam, force-quit,
|
|
// suspended mid-utterance). Its own process is gone, so this side is the
|
|
// only one that can record it.
|
|
persistStartupFailure("hostDisconnected")
|
|
if let id = currentUtteranceId,
|
|
cancelledDictationUtteranceIDs.remove(id) != nil {
|
|
stopUtteranceCountdown()
|
|
stopFlowWatchdog()
|
|
ExtensionScreenWakeLock.release()
|
|
resetEditTransportState()
|
|
state.level = 0
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
traceState(
|
|
"dictation.cancelledHostDisconnected",
|
|
extra: "utterance=\(id.uuidString.prefix(8))"
|
|
)
|
|
return
|
|
}
|
|
if let id = currentUtteranceId,
|
|
cancelledAIUtteranceIDs.remove(id) != nil {
|
|
stopUtteranceCountdown()
|
|
stopFlowWatchdog()
|
|
ExtensionScreenWakeLock.release()
|
|
resetEditTransportState()
|
|
state.level = 0
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
if deliverRawFallbackIfAvailable(reason: "hostDisconnected") {
|
|
return
|
|
}
|
|
traceState("hostDisconnected.fail")
|
|
isAwaitingFlowResult = false
|
|
isFlowRecording = false
|
|
isPendingFlowStart = false
|
|
recordAfterHandoff = false
|
|
stopUtteranceCountdown()
|
|
ExtensionScreenWakeLock.release()
|
|
writeCommand(.abort)
|
|
currentUtteranceId = nil
|
|
lastStoppedUtteranceId = nil
|
|
stopFlowWatchdog()
|
|
state.level = 0
|
|
let message = ExtL10n.string("keyboard.flow.hostDisconnected")
|
|
if currentUtteranceRequest?.isEdit == true {
|
|
onEditFailure(message)
|
|
resetEditTransportState()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
if currentUtteranceRequest?.isAIQuestion == true {
|
|
onAIFailure(message, currentUtteranceId)
|
|
resetEditTransportState()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
state.phase = .error(.flowSessionExpired, message: message)
|
|
scheduleAutoClearError()
|
|
recomputeMicVoiceAvailability()
|
|
debug("host disconnected while awaiting Flow result")
|
|
}
|
|
|
|
private func resetEditTransportState() {
|
|
isAwaitingFlowResult = false
|
|
isFlowRecording = false
|
|
isPendingFlowStart = false
|
|
recordAfterHandoff = false
|
|
currentUtteranceId = nil
|
|
currentStartDeadlineAt = nil
|
|
currentUtteranceRequest = nil
|
|
editHostConfirmed = false
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
FlowSessionBridge.clearStartTransaction()
|
|
}
|
|
|
|
@discardableResult
|
|
private func deliverRawFallbackIfAvailable(reason: String) -> Bool {
|
|
FlowSessionBridge.reloadFromDisk()
|
|
guard let result = matchingResult(),
|
|
result.allowsRawFallback,
|
|
result.status == .partial
|
|
|| result.status == .rawReady
|
|
|| (result.status == .final && result.rawText != nil),
|
|
let raw = (result.rawText ?? result.text)?
|
|
.trimmingCharacters(in: .whitespacesAndNewlines),
|
|
!raw.isEmpty else {
|
|
return false
|
|
}
|
|
textInserter.handleFlowTranscript(
|
|
TranscriptionDelivery(text: raw, polishWarning: nil)
|
|
)
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision
|
|
)
|
|
)
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
lastConsumedUtteranceId = result.utteranceId
|
|
lastStoppedUtteranceId = nil
|
|
currentUtteranceId = nil
|
|
isAwaitingFlowResult = false
|
|
isFlowRecording = false
|
|
stopFlowWatchdog()
|
|
state.level = 0
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
FlowTrace.transcript(
|
|
"keyboard.insert",
|
|
raw,
|
|
"via=rawFallback reason=\(reason) utterance=\(result.utteranceId.uuidString.prefix(8))"
|
|
)
|
|
return true
|
|
}
|
|
|
|
private func showFlowSessionExpiredHint() {
|
|
persistStartupFailure("sessionExpired")
|
|
let message = ExtL10n.string("keyboard.flow.sessionExpired")
|
|
state.phase = .error(.flowSessionExpired, message: message)
|
|
scheduleAutoClearError()
|
|
recomputeMicVoiceAvailability()
|
|
}
|
|
|
|
private func showManualOpenHint(path: String) {
|
|
let msg: String
|
|
if !hasFullAccess() {
|
|
msg = ExtL10n.string("keyboard.error.fullAccessForJump")
|
|
} else if path == "settings" {
|
|
msg = ExtL10n.string("keyboard.error.manualOpenSettings")
|
|
} else if path == "startflow" {
|
|
msg = ExtL10n.string("keyboard.error.manualOpenForFlow")
|
|
} else {
|
|
msg = ExtL10n.string("keyboard.error.manualOpenSettings")
|
|
}
|
|
state.phase = .error(.manualOpenRequired, message: msg)
|
|
scheduleAutoClearError()
|
|
recomputeMicVoiceAvailability()
|
|
}
|
|
|
|
/// Scheme C: voice needs host-app setup; typing stays available.
|
|
private func promptFinishSetupInApp() {
|
|
let msg = ExtL10n.string("keyboard.hint.finishSetupInApp")
|
|
state.phase = .error(.manualOpenRequired, message: msg)
|
|
scheduleAutoClearError()
|
|
recomputeMicVoiceAvailability()
|
|
openHostApp("settings")
|
|
traceState("onboarding.incomplete", extra: "action=openHostApp(settings)")
|
|
}
|
|
|
|
private func startFlowRecording() {
|
|
if let deadline = currentStartDeadlineAt,
|
|
Date().timeIntervalSince1970 >= deadline {
|
|
// The 8 s budget started at the mic press and was spent waiting for
|
|
// the host; recording never even got attempted.
|
|
persistStartupFailure("startBudgetExpiredBeforeRecord")
|
|
if currentUtteranceRequest?.isEdit == true {
|
|
onEditFailure(ExtL10n.string("keyboard.edit.error.startTimeout"))
|
|
abortEditRecording()
|
|
} else if currentUtteranceRequest?.isAIQuestion == true {
|
|
onAIFailure(
|
|
ExtL10n.string("keyboard.ai.error.startTimeout"),
|
|
currentUtteranceId
|
|
)
|
|
cancelAIRecording()
|
|
} else {
|
|
state.phase = .error(
|
|
.hostAudioUnavailable,
|
|
message: ExtL10n.string("keyboard.flow.resultTimeout")
|
|
)
|
|
scheduleAutoClearError()
|
|
currentStartDeadlineAt = nil
|
|
}
|
|
return
|
|
}
|
|
recomputeMicVoiceAvailability()
|
|
let withinReadyGrace = lastHostReadyAt > 0
|
|
&& (Date().timeIntervalSince1970 - lastHostReadyAt) <= Self.hostReadyGrace
|
|
if !state.micVoiceAvailability.isReady {
|
|
let action = FlowHandoffPolicy.micPressAction(
|
|
availability: state.micVoiceAvailability,
|
|
sessionActive: FlowSessionBridge.isSessionActive(),
|
|
hostReachable: FlowSessionBridge.isHostReachable(),
|
|
hostStale: FlowSessionBridge.isHostStale(),
|
|
withinReadyGrace: withinReadyGrace
|
|
)
|
|
traceState(
|
|
"startFlowRecording.blocked",
|
|
extra: "availability=\(String(describing: state.micVoiceAvailability)) action=\(action)"
|
|
)
|
|
switch action {
|
|
case .waitForHostReady(let recordWhenReady):
|
|
recordWhenHostReady = recordWhenReady
|
|
startHostReadyWaitIfNeeded()
|
|
case .openHostColdStart:
|
|
beginFlowStart(recordAfterHandoff: true)
|
|
case .startRecording, .ignore:
|
|
break
|
|
}
|
|
return
|
|
}
|
|
isPendingFlowStart = false
|
|
flowStartDeadline = 0
|
|
stopFlowWatchdog()
|
|
|
|
guard let sessionId = FlowSessionBridge.readySnapshot()?.sessionId else {
|
|
traceState("startFlowRecording.blocked", extra: "reason=missingSessionIdInReadySnapshot")
|
|
// Snapshot lag with a live session → wait; only cold-start if host is dead.
|
|
if FlowHandoffPolicy.shouldOpenHostColdStart(
|
|
sessionActive: FlowSessionBridge.isSessionActive(),
|
|
hostReachable: FlowSessionBridge.isHostReachable(),
|
|
hostStale: FlowSessionBridge.isHostStale(),
|
|
withinReadyGrace: withinReadyGrace
|
|
) {
|
|
beginFlowStart(recordAfterHandoff: true)
|
|
} else {
|
|
recordWhenHostReady = true
|
|
startHostReadyWaitIfNeeded()
|
|
}
|
|
return
|
|
}
|
|
|
|
activeSessionId = sessionId
|
|
if currentUtteranceId == nil {
|
|
currentUtteranceId = UUID()
|
|
}
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(currentUtteranceId)
|
|
lastStoppedUtteranceId = nil
|
|
|
|
// Prefilled AI hint: skip mic / ASR and ask the host to answer text.
|
|
if currentUtteranceRequest?.isAIQuestion == true,
|
|
let question = currentUtteranceRequest?.aiQuestionText?
|
|
.trimmingCharacters(in: .whitespacesAndNewlines),
|
|
!question.isEmpty {
|
|
writeSubmitAIQuestion(question)
|
|
isFlowRecording = false
|
|
isAwaitingFlowResult = true
|
|
state.lastTranscript = ""
|
|
state.phase = .processing
|
|
if let currentUtteranceId {
|
|
onAIGeneratingStarted(currentUtteranceId)
|
|
}
|
|
startFlowResultWatchdog()
|
|
recomputeMicVoiceAvailability()
|
|
traceState("startFlowRecording.submitAIQuestion")
|
|
return
|
|
}
|
|
|
|
writeCommand(.startRecording)
|
|
isFlowRecording = true
|
|
state.lastTranscript = ""
|
|
state.phase = currentUtteranceRequest?.isEdit == true
|
|
? .requestingPermissions
|
|
: .recording
|
|
if currentUtteranceRequest?.isAIQuestion == true,
|
|
let currentUtteranceId {
|
|
onAIRecordingStarted(currentUtteranceId)
|
|
}
|
|
recomputeMicVoiceAvailability()
|
|
if let view = wakeLockView() {
|
|
ExtensionScreenWakeLock.acquire(from: view)
|
|
}
|
|
if currentUtteranceRequest?.isEdit != true {
|
|
startUtteranceCountdown()
|
|
}
|
|
startFlowLevelWatchdog()
|
|
traceState("startFlowRecording.started")
|
|
}
|
|
|
|
private func writeSubmitAIQuestion(_ text: String) {
|
|
guard let activeSessionId, let currentUtteranceId else { return }
|
|
let command = FlowCommand(
|
|
sessionId: activeSessionId,
|
|
utteranceId: currentUtteranceId,
|
|
commandSeq: nextCommandSeq(),
|
|
action: .submitAIQuestion,
|
|
localeId: state.localeId,
|
|
utteranceMode: .aiQuestion,
|
|
aiConversationID: currentUtteranceRequest?.aiConversationID,
|
|
aiQuestionText: text,
|
|
aiWebPageURL: currentUtteranceRequest?.aiWebPageURL,
|
|
aiTaskKind: currentUtteranceRequest?.aiTaskKind,
|
|
managedRequestSource: currentUtteranceRequest?.managedRequestSource,
|
|
managedRequestPurpose: currentUtteranceRequest?.managedRequestPurpose,
|
|
managedOOBEFeature: currentUtteranceRequest?.managedOOBEFeature,
|
|
aiThinkingEnabled: currentUtteranceRequest?.aiThinkingEnabled,
|
|
startDeadlineAt: currentStartDeadlineAt
|
|
)
|
|
FlowSessionBridge.writeCommand(command)
|
|
if let currentStartDeadlineAt {
|
|
FlowSessionBridge.writeStartTransaction(
|
|
FlowStartTransaction(
|
|
sessionID: activeSessionId,
|
|
utteranceID: currentUtteranceId,
|
|
deadlineAt: currentStartDeadlineAt,
|
|
phase: .issued
|
|
)
|
|
)
|
|
}
|
|
FlowTrace.keyboard(
|
|
"command.submitAIQuestion",
|
|
"seq=\(command.commandSeq) utterance=\(currentUtteranceId.uuidString.prefix(8)) "
|
|
+ "chars=\(text.count)"
|
|
)
|
|
}
|
|
|
|
private func startUtteranceCountdown() {
|
|
utteranceStartedAt = Date().timeIntervalSince1970
|
|
state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration)
|
|
utteranceTimerTask?.cancel()
|
|
utteranceTimerTask = Task { @MainActor [weak self] in
|
|
while let self, self.isFlowRecording, !Task.isCancelled {
|
|
let elapsed = Date().timeIntervalSince1970 - self.utteranceStartedAt
|
|
let remaining = max(0, Int(ceil(FlowSessionKeys.maxUtteranceDuration - elapsed)))
|
|
self.state.utteranceRemainingSeconds = remaining
|
|
if remaining <= 0 {
|
|
self.pressEnded()
|
|
return
|
|
}
|
|
try? await Task.sleep(nanoseconds: 200_000_000)
|
|
}
|
|
}
|
|
}
|
|
|
|
private func stopUtteranceCountdown() {
|
|
utteranceTimerTask?.cancel()
|
|
utteranceTimerTask = nil
|
|
state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration)
|
|
}
|
|
|
|
private func cancelPendingFlowStart() {
|
|
isPendingFlowStart = false
|
|
recordAfterHandoff = false
|
|
recordWhenHostReady = false
|
|
flowStartDeadline = 0
|
|
coldStartDebouncer.reset()
|
|
stopFlowWatchdog()
|
|
stopHostReadyWait()
|
|
state.phase = .idle
|
|
state.lastTranscript = ""
|
|
recomputeMicVoiceAvailability()
|
|
traceState("pendingStart.cancelledByUser")
|
|
}
|
|
|
|
private func startFlowStartWatchdog() {
|
|
stopFlowWatchdog()
|
|
flowWatchdogTask = Task { @MainActor [weak self] in
|
|
while let self, !Task.isCancelled, self.isPendingFlowStart {
|
|
self.recomputeMicVoiceAvailability()
|
|
if FlowSessionBridge.isHostReady() {
|
|
self.completeFlowStartHandoff()
|
|
return
|
|
}
|
|
let now = Date().timeIntervalSince1970
|
|
// The host acknowledged the request and is alive & warming
|
|
// (cold launch / foreground handoff). Extend the soft budget up
|
|
// to the hard cap so a legitimate cold start is not failed at
|
|
// the initial launch budget while the host is provably working.
|
|
if let ackReason = FlowSessionBridge.hostStartInProgress() {
|
|
let extended = min(
|
|
now + FlowSessionKeys.hostStartAckExtension,
|
|
self.flowStartHardDeadline
|
|
)
|
|
if extended > self.flowStartDeadline {
|
|
self.flowStartDeadline = extended
|
|
self.currentStartDeadlineAt = extended
|
|
}
|
|
// Host is up but parked waiting for the user to bring it to
|
|
// the foreground — PiP cannot arm from the background. Prompt
|
|
// once so the user can tap back instead of watching a silent
|
|
// spinner run out the clock. Edit mode has no manual-open UI,
|
|
// so keep letting it wait out the hard cap.
|
|
if ackReason == .waitingForForeground,
|
|
!self.coldStartForegroundHintShown,
|
|
self.currentUtteranceRequest?.isEdit != true {
|
|
self.coldStartForegroundHintShown = true
|
|
self.showManualOpenHint(path: "startflow")
|
|
}
|
|
}
|
|
if self.flowStartDeadline > 0, now > self.flowStartDeadline {
|
|
self.isPendingFlowStart = false
|
|
self.recordAfterHandoff = false
|
|
self.traceState("startWatchdog.timeout")
|
|
// The single most common user-visible failure: the host was
|
|
// asked to start and never published ready inside the
|
|
// budget. Persist before clearing the deadline so the
|
|
// report still carries it.
|
|
self.persistStartupFailure("startTimeout")
|
|
self.flowStartDeadline = 0
|
|
if self.currentUtteranceRequest?.isEdit == true {
|
|
self.onEditFailure(
|
|
ExtL10n.string("keyboard.edit.error.startTimeout")
|
|
)
|
|
self.abortEditRecording()
|
|
} else {
|
|
self.showManualOpenHint(path: "startflow")
|
|
}
|
|
return
|
|
}
|
|
try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs)
|
|
}
|
|
}
|
|
}
|
|
|
|
private func completeFlowStartHandoff() {
|
|
let shouldRecord = recordAfterHandoff
|
|
isPendingFlowStart = false
|
|
recordAfterHandoff = false
|
|
flowStartDeadline = 0
|
|
flowStartHardDeadline = 0
|
|
coldStartForegroundHintShown = false
|
|
stopFlowWatchdog()
|
|
state.lastTranscript = ""
|
|
refreshSessionState()
|
|
if shouldRecord {
|
|
startFlowRecording()
|
|
traceState("completeFlowStartHandoff.done", extra: "record=1")
|
|
} else {
|
|
recomputeMicVoiceAvailability()
|
|
traceState("completeFlowStartHandoff.done", extra: "record=0 warmOnly")
|
|
}
|
|
}
|
|
|
|
private func startFlowLevelWatchdog() {
|
|
stopFlowWatchdog()
|
|
flowWatchdogTask = Task { @MainActor [weak self] in
|
|
while let self, !Task.isCancelled, self.isFlowRecording {
|
|
let levels = FlowSessionBridge.audioLevels()
|
|
if let peak = levels.max(), peak > 0 {
|
|
self.state.level = Double(peak)
|
|
}
|
|
self.refreshFlowPartialIfNeeded()
|
|
let staleness = FlowSessionBridge.heartbeatStaleness() ?? .infinity
|
|
if staleness > 5 {
|
|
self.debug("levelWatchdog: host heartbeat stale while recording")
|
|
self.failHostDisconnected()
|
|
return
|
|
}
|
|
try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs)
|
|
}
|
|
}
|
|
}
|
|
|
|
private func refreshFlowPartialIfNeeded() {
|
|
guard isFlowRecording || isAwaitingFlowResult else { return }
|
|
switch state.phase {
|
|
case .recording, .processing:
|
|
guard let result = matchingResult() else { return }
|
|
if result.status == .streaming,
|
|
result.resolvedUtteranceMode == .aiQuestion {
|
|
onAIStreamingAnswer(result.text ?? "", result.utteranceId)
|
|
return
|
|
}
|
|
if result.status == .partial || result.status == .rawReady,
|
|
let partial = result.text,
|
|
!partial.isEmpty {
|
|
if result.resolvedUtteranceMode == .aiQuestion,
|
|
AIClipboardPrompt.isInternalPrompt(partial) {
|
|
onAIGeneratingStarted(result.utteranceId)
|
|
return
|
|
}
|
|
state.lastTranscript = partial
|
|
if result.resolvedUtteranceMode == .aiQuestion {
|
|
onAITranscript(partial, result.utteranceId, result.status)
|
|
}
|
|
}
|
|
default:
|
|
break
|
|
}
|
|
}
|
|
|
|
private func startFlowResultWatchdog(
|
|
purpose: ResultWatchdogPurpose = .delivery
|
|
) {
|
|
stopFlowWatchdog()
|
|
isAwaitingFlowResult = true
|
|
let startedAt = Date().timeIntervalSince1970
|
|
let resultTimeout: TimeInterval
|
|
if purpose == .cancellation {
|
|
resultTimeout = FlowSessionKeys.utteranceStartBudget
|
|
} else if currentUtteranceRequest?.isAIQuestion == true {
|
|
resultTimeout = FlowSessionKeys.keyboardAIResultTimeout(
|
|
engineMode: state.engineMode
|
|
)
|
|
} else if currentUtteranceRequest?.isEdit == true {
|
|
resultTimeout = FlowSessionKeys.editLastInputProcessingBudget
|
|
} else {
|
|
resultTimeout = FlowWatchdog.resultTimeout(engineMode: state.engineMode)
|
|
}
|
|
debug("resultWatchdog started timeout=\(Int(resultTimeout))s engine=\(state.engineMode)")
|
|
flowWatchdogTask = Task { @MainActor [weak self] in
|
|
while let self, !Task.isCancelled {
|
|
FlowSessionBridge.reloadFromDisk()
|
|
if let result = self.matchingResult(),
|
|
self.discardUnsupportedLegacyResultIfNeeded(result) {
|
|
return
|
|
}
|
|
if let result = self.matchingResult(),
|
|
self.consumeCancelledDictationResultIfNeeded(result) {
|
|
return
|
|
}
|
|
if let result = self.matchingResult(),
|
|
self.consumeDiscardedEmptyResultIfNeeded(result) {
|
|
return
|
|
}
|
|
if let result = self.matchingResult(),
|
|
self.consumeCancelledEditResultIfNeeded(result) {
|
|
return
|
|
}
|
|
if let result = self.matchingResult(),
|
|
self.consumeCancelledAIResultIfNeeded(result) {
|
|
return
|
|
}
|
|
if let result = self.matchingResult(), result.status == .final, let text = result.text, !text.isEmpty {
|
|
self.isAwaitingFlowResult = false
|
|
self.stopFlowWatchdog()
|
|
if result.resolvedUtteranceMode == .editLastInput {
|
|
self.onEditResult(result)
|
|
return
|
|
}
|
|
if result.resolvedUtteranceMode == .aiQuestion {
|
|
self.completeAIResult(result)
|
|
return
|
|
}
|
|
self.textInserter.handleFlowTranscript(
|
|
TranscriptionDelivery(
|
|
text: text,
|
|
polishWarning: result.warning,
|
|
historyEntryID: result.historyEntryID,
|
|
historyEntryRevision: result.historyEntryRevision
|
|
)
|
|
)
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision
|
|
)
|
|
)
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
self.lastConsumedUtteranceId = result.utteranceId
|
|
self.lastStoppedUtteranceId = nil
|
|
self.currentUtteranceId = nil
|
|
self.debug("resultWatchdog consumed delivery len=\(text.count)")
|
|
FlowTrace.transcript(
|
|
"keyboard.insert",
|
|
text,
|
|
"via=resultWatchdog utterance=\(result.utteranceId.uuidString.prefix(8)) "
|
|
+ "commandSeq=\(result.commandSeq) "
|
|
+ "waitedSeconds=\(String(format: "%.2f", Date().timeIntervalSince1970 - startedAt))"
|
|
)
|
|
return
|
|
}
|
|
if let result = self.matchingResult(), self.isTerminalFailure(result) {
|
|
self.isAwaitingFlowResult = false
|
|
self.stopFlowWatchdog()
|
|
if result.resolvedUtteranceMode == .editLastInput {
|
|
self.onEditFailure(
|
|
result.text ?? ExtL10n.string("keyboard.edit.error.processing")
|
|
)
|
|
self.completeEditResult(result, outcome: .rejected)
|
|
self.state.phase = .idle
|
|
return
|
|
}
|
|
if result.resolvedUtteranceMode == .aiQuestion {
|
|
self.completeAIFailure(result)
|
|
return
|
|
}
|
|
FlowSessionBridge.writeAck(
|
|
FlowAck(
|
|
sessionId: result.sessionId,
|
|
utteranceId: result.utteranceId,
|
|
commandSeq: result.commandSeq,
|
|
hostGeneration: result.hostGeneration,
|
|
revision: result.revision
|
|
)
|
|
)
|
|
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
|
self.lastConsumedUtteranceId = result.utteranceId
|
|
self.lastStoppedUtteranceId = nil
|
|
self.currentUtteranceId = nil
|
|
let error = FlowTranscriptionError(
|
|
message: result.text ?? ExtL10n.string("keyboard.flow.resultTimeout"),
|
|
kind: result.errorKind ?? .generic
|
|
)
|
|
self.debug("resultWatchdog consumed error kind=\(error.kind.rawValue)")
|
|
FlowTrace.warn(
|
|
"keyboard.resultFailed",
|
|
"via=resultWatchdog status=\(result.status.rawValue) "
|
|
+ "kind=\(error.kind.rawValue) "
|
|
+ "utterance=\(result.utteranceId.uuidString.prefix(8)) "
|
|
+ "messageLen=\(error.message.count)"
|
|
)
|
|
self.state.phase = .error(
|
|
.fromFlowTranscription(error),
|
|
message: error.message
|
|
)
|
|
self.scheduleAutoClearError()
|
|
self.recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
self.refreshFlowPartialIfNeeded()
|
|
let now = Date().timeIntervalSince1970
|
|
let staleness = FlowSessionBridge.heartbeatStaleness() ?? .infinity
|
|
if self.isFlowRecording, staleness > 5 {
|
|
self.debug("level/result watchdog: host heartbeat stale while recording")
|
|
self.failHostDisconnected()
|
|
return
|
|
}
|
|
if staleness > FlowSessionKeys.heartbeatZombieInterval {
|
|
self.debug("resultWatchdog: host heartbeat zombie (staleness=\(String(format: "%.1f", staleness))s)")
|
|
self.failHostDisconnected()
|
|
return
|
|
}
|
|
if !FlowSessionBridge.isHostReachable(),
|
|
now - startedAt > FlowSessionKeys.keyboardHostDisconnectFailFast {
|
|
self.debug("resultWatchdog: host unreachable after \(String(format: "%.1f", now - startedAt))s")
|
|
self.failHostDisconnected()
|
|
return
|
|
}
|
|
if now - startedAt > resultTimeout {
|
|
if let id = self.currentUtteranceId,
|
|
self.cancelledDictationUtteranceIDs.remove(id) != nil {
|
|
self.resetEditTransportState()
|
|
self.state.level = 0
|
|
self.state.phase = .idle
|
|
self.state.lastTranscript = ""
|
|
self.recomputeMicVoiceAvailability()
|
|
self.traceState(
|
|
"dictation.cancelledResultTimeout",
|
|
extra: "utterance=\(id.uuidString.prefix(8))"
|
|
)
|
|
return
|
|
}
|
|
if let id = self.currentUtteranceId,
|
|
self.cancelledEditUtteranceIDs.remove(id) != nil {
|
|
self.resetEditTransportState()
|
|
self.state.phase = .idle
|
|
self.state.lastTranscript = ""
|
|
self.recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
if let id = self.currentUtteranceId,
|
|
self.cancelledAIUtteranceIDs.remove(id) != nil {
|
|
self.resetEditTransportState()
|
|
self.state.phase = .idle
|
|
self.state.lastTranscript = ""
|
|
self.recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
if self.currentUtteranceRequest?.isEdit == true {
|
|
self.isAwaitingFlowResult = false
|
|
self.stopFlowWatchdog()
|
|
self.writeCommand(.abort)
|
|
self.onEditFailure(
|
|
ExtL10n.string("keyboard.edit.error.processingTimeout")
|
|
)
|
|
self.currentUtteranceId = nil
|
|
self.currentStartDeadlineAt = nil
|
|
self.currentUtteranceRequest = nil
|
|
self.editHostConfirmed = false
|
|
self.state.phase = .idle
|
|
return
|
|
}
|
|
if self.currentUtteranceRequest?.isAIQuestion == true {
|
|
let utteranceID = self.currentUtteranceId
|
|
self.isAwaitingFlowResult = false
|
|
self.stopFlowWatchdog()
|
|
self.writeCommand(.abort)
|
|
self.onAIFailure(
|
|
ExtL10n.string("keyboard.ai.error.requestTimeout"),
|
|
utteranceID
|
|
)
|
|
self.resetEditTransportState()
|
|
self.state.phase = .idle
|
|
self.state.lastTranscript = ""
|
|
self.recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
if self.deliverRawFallbackIfAvailable(reason: "resultTimeout") {
|
|
return
|
|
}
|
|
self.isAwaitingFlowResult = false
|
|
self.stopFlowWatchdog()
|
|
self.currentUtteranceId = nil
|
|
self.lastStoppedUtteranceId = nil
|
|
self.debug("resultWatchdog TIMEOUT after \(Int(resultTimeout))s — no result from host")
|
|
let msg = ExtL10n.string("keyboard.flow.resultTimeout")
|
|
self.state.phase = .error(.flowResultTimeout, message: msg)
|
|
self.scheduleAutoClearError()
|
|
self.recomputeMicVoiceAvailability()
|
|
return
|
|
}
|
|
try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs)
|
|
}
|
|
}
|
|
}
|
|
|
|
private func stopFlowWatchdog() {
|
|
flowWatchdogTask?.cancel()
|
|
flowWatchdogTask = nil
|
|
}
|
|
|
|
private func debug(_ message: String) {
|
|
// Also feeds the shared breadcrumb window, so a terminal start failure
|
|
// on this side ships with the keyboard's own timeline instead of only
|
|
// whatever the host happened to record.
|
|
FlowFailureDiagnostics.record(message)
|
|
OSGLog.keyboardExt.info("\(message, privacy: .public)")
|
|
}
|
|
|
|
/// Writes a startup-failure report for a keyboard-observed dead end and
|
|
/// asks the host for its companion window. This is the capture path that
|
|
/// survives the host never launching at all.
|
|
private func persistStartupFailure(_ reason: String) {
|
|
FlowFailureDiagnostics.persistKeyboardStartupFailure(
|
|
reason: reason,
|
|
context: keyboardStartupFailureContext()
|
|
)
|
|
}
|
|
|
|
private func keyboardStartupFailureContext() -> [String: String] {
|
|
let now = Date().timeIntervalSince1970
|
|
return [
|
|
"pendingFlowStart": isPendingFlowStart ? "true" : "false",
|
|
"startDeadlineRemainingSeconds": flowStartDeadline > 0
|
|
? String(format: "%.3f", flowStartDeadline - now)
|
|
: "nil",
|
|
"startBudgetSeconds": String(format: "%.1f", FlowWatchdog.startTimeout),
|
|
"recordAfterHandoff": recordAfterHandoff ? "true" : "false",
|
|
"flowRecording": isFlowRecording ? "true" : "false",
|
|
"awaitingResult": isAwaitingFlowResult ? "true" : "false",
|
|
"sessionProvenReady": sessionProvenReady ? "true" : "false",
|
|
"fullAccess": hasFullAccess() ? "true" : "false",
|
|
"voiceSetupReady": voiceSetupReady ? "true" : "false",
|
|
"requestKind": currentUtteranceRequest?.isEdit == true
|
|
? "edit"
|
|
: (currentUtteranceRequest?.isAIQuestion == true ? "aiQuestion" : "dictation"),
|
|
"keyboardPhase": String(describing: state.phase)
|
|
]
|
|
}
|
|
|
|
private func traceState(_ event: String, extra: String? = nil) {
|
|
let staleness = FlowSessionBridge.heartbeatStaleness().map { String(format: "%.1f", $0) } ?? "nil"
|
|
let sessionId = activeSessionId?.uuidString ?? "nil"
|
|
let utteranceId = currentUtteranceId?.uuidString ?? "nil"
|
|
let summary = [
|
|
"event=\(event)",
|
|
"phase=\(String(describing: state.phase))",
|
|
"availability=\(String(describing: state.micVoiceAvailability))",
|
|
"pending=\(isPendingFlowStart)",
|
|
"recording=\(isFlowRecording)",
|
|
"awaiting=\(isAwaitingFlowResult)",
|
|
"sessionId=\(sessionId)",
|
|
"utteranceId=\(utteranceId)",
|
|
"cmdSeq=\(currentCommandSeq)",
|
|
"sessionActive=\(FlowSessionBridge.isSessionActive())",
|
|
"hostReady=\(FlowSessionBridge.isHostReady())",
|
|
"heartbeatStaleness=\(staleness)"
|
|
].joined(separator: " ")
|
|
if let extra, !extra.isEmpty {
|
|
debug("[trace] \(summary) \(extra)")
|
|
} else {
|
|
debug("[trace] \(summary)")
|
|
}
|
|
}
|
|
}
|