feat(keyboard): ship AI mode surface with streaming search answers
Add the AI keyboard tab, Agent settings, and user-owned LLM key path for 1.7.0, including streaming answers and web-search transports without the built-in DeepSeek fallback.
This commit is contained in:
@@ -0,0 +1,162 @@
|
||||
// AIKeyboardCoordinator.swift
|
||||
// OSGKeyboard · Keyboard Extension
|
||||
//
|
||||
// Owns the temporary AI-mode UI state. Audio, ASR, and LLM work remain in the
|
||||
// shared Flow transport and host app; this coordinator never performs network
|
||||
// work and never inserts an answer before explicit user confirmation.
|
||||
|
||||
import Foundation
|
||||
import OSGKeyboardShared
|
||||
|
||||
@MainActor
|
||||
final class AIKeyboardCoordinator {
|
||||
private let state: KeyboardState
|
||||
private let flow: KeyboardFlowCoordinator
|
||||
private let insertAnswer: (AIAnswer) -> Bool
|
||||
private let performReturn: () -> Void
|
||||
|
||||
init(
|
||||
state: KeyboardState,
|
||||
flow: KeyboardFlowCoordinator,
|
||||
insertAnswer: @escaping (AIAnswer) -> Bool,
|
||||
performReturn: @escaping () -> Void
|
||||
) {
|
||||
self.state = state
|
||||
self.flow = flow
|
||||
self.insertAnswer = insertAnswer
|
||||
self.performReturn = performReturn
|
||||
}
|
||||
|
||||
func beginNewPresentation() {
|
||||
endConversationIfNeeded()
|
||||
state.aiSession.enter()
|
||||
}
|
||||
|
||||
func enterIfNeeded() {
|
||||
guard !state.aiSession.isActive else { return }
|
||||
state.aiSession.enter()
|
||||
}
|
||||
|
||||
func leave() {
|
||||
if state.aiSession.isBusy {
|
||||
flow.cancelAIRecording()
|
||||
}
|
||||
endConversationIfNeeded()
|
||||
state.aiSession.leave()
|
||||
}
|
||||
|
||||
func toggleMicrophone() {
|
||||
switch state.aiSession.phase {
|
||||
case .listening:
|
||||
guard let utteranceID = state.aiSession.activeUtteranceID else { return }
|
||||
state.aiSession.beginRecognizing(utteranceID: utteranceID)
|
||||
flow.stopAIRecording()
|
||||
case .idle, .ready, .awaitingSend, .inserted, .sent, .failed:
|
||||
enterIfNeeded()
|
||||
guard let conversationID = state.aiSession.conversationID else { return }
|
||||
let disposition = flow.beginAIRecording(conversationID: conversationID)
|
||||
if case .rejected(let rejection) = disposition {
|
||||
state.aiSession.fail(message(for: rejection), utteranceID: nil)
|
||||
}
|
||||
case .inactive:
|
||||
enterIfNeeded()
|
||||
toggleMicrophone()
|
||||
case .preparing, .recognizing, .generating:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
guard state.aiSession.isBusy else { return }
|
||||
flow.cancelAIRecording()
|
||||
state.aiSession.cancelCurrentWork()
|
||||
}
|
||||
|
||||
func sendLatestAnswer() {
|
||||
if state.aiSession.canInsert, let answer = state.aiSession.answer {
|
||||
guard insertAnswer(answer) else { return }
|
||||
state.aiSession.markAnswerInserted(
|
||||
offersSend: state.returnKeyRole == .send
|
||||
)
|
||||
} else if state.aiSession.canSend {
|
||||
state.aiSession.markAnswerSent()
|
||||
// Let the host consume the inserted answer before issuing Return.
|
||||
Task { @MainActor [weak self] in
|
||||
await Task.yield()
|
||||
self?.performReturn()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func utterancePrepared(_ utteranceID: UUID) {
|
||||
state.aiSession.beginPreparing(utteranceID: utteranceID)
|
||||
}
|
||||
|
||||
func recordingStarted(_ utteranceID: UUID) {
|
||||
state.aiSession.beginListening(utteranceID: utteranceID)
|
||||
}
|
||||
|
||||
func recognitionStarted(_ utteranceID: UUID) {
|
||||
state.aiSession.beginRecognizing(utteranceID: utteranceID)
|
||||
}
|
||||
|
||||
func receiveTranscript(
|
||||
_ transcript: String,
|
||||
utteranceID: UUID,
|
||||
status: FlowResult.Status
|
||||
) {
|
||||
state.aiSession.updateTranscript(transcript, utteranceID: utteranceID)
|
||||
if status == .rawReady {
|
||||
state.aiSession.beginGenerating(
|
||||
question: transcript,
|
||||
utteranceID: utteranceID
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func receivePartialAnswer(_ draft: String, utteranceID: UUID) {
|
||||
state.aiSession.receivePartialAnswer(draft, utteranceID: utteranceID)
|
||||
}
|
||||
|
||||
func receive(result: FlowResult) {
|
||||
guard result.resolvedUtteranceMode == .aiQuestion else {
|
||||
return
|
||||
}
|
||||
guard result.aiConversationID == state.aiSession.conversationID,
|
||||
let answer = result.text,
|
||||
!answer.isEmpty else {
|
||||
state.aiSession.fail(
|
||||
ExtL10n.string("keyboard.ai.error.requestFailed"),
|
||||
utteranceID: result.utteranceId
|
||||
)
|
||||
return
|
||||
}
|
||||
state.aiSession.receiveAnswer(answer, utteranceID: result.utteranceId)
|
||||
}
|
||||
|
||||
func fail(_ message: String, utteranceID: UUID?) {
|
||||
state.aiSession.fail(message, utteranceID: utteranceID)
|
||||
}
|
||||
|
||||
private func endConversationIfNeeded() {
|
||||
guard let conversationID = state.aiSession.conversationID else { return }
|
||||
flow.endAIConversation(conversationID)
|
||||
}
|
||||
|
||||
private func message(for rejection: FlowUtteranceStartRejection) -> String {
|
||||
switch rejection {
|
||||
case .onboardingIncomplete:
|
||||
return ExtL10n.string("keyboard.hint.finishSetupInApp")
|
||||
case .missingAPIKey:
|
||||
return ExtL10n.string("keyboard.ai.error.missingAPIKey")
|
||||
case .noFullAccess:
|
||||
return ExtL10n.string("keyboard.error.fullAccessRequired")
|
||||
case .appGroupUnavailable:
|
||||
return ExtL10n.string("keyboard.error.appGroupCommunication")
|
||||
case .hostUnavailable:
|
||||
return ExtL10n.string("keyboard.flow.hostDisconnected")
|
||||
case .pipelineBusy:
|
||||
return ExtL10n.string("keyboard.ai.error.pipelineBusy")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -42,10 +42,7 @@ public struct AppGroupPersistor {
|
||||
state.handednessPreference = store.handednessPreference
|
||||
state.cursorDragNavigationEnabled = store.cursorDragNavigationEnabled
|
||||
state.keyboardHapticIntensity = store.keyboardHapticIntensity
|
||||
state.micDisabled = store.isCloudAPIKeyMissingForVoiceInput
|
||||
state.micDisabledHint = store.isCloudAPIKeyMissingForVoiceInput
|
||||
? ExtL10n.string("keyboard.mic.disabled.missingApiKey")
|
||||
: ""
|
||||
applyAPIKeyAvailability(store: store, into: state)
|
||||
|
||||
#if DEBUG
|
||||
// Print a masked view of the live App Group config so we can see
|
||||
@@ -95,10 +92,26 @@ public struct AppGroupPersistor {
|
||||
state.handednessPreference = store.handednessPreference
|
||||
state.cursorDragNavigationEnabled = store.cursorDragNavigationEnabled
|
||||
state.keyboardHapticIntensity = store.keyboardHapticIntensity
|
||||
state.micDisabled = store.isCloudAPIKeyMissingForVoiceInput
|
||||
state.micDisabledHint = store.isCloudAPIKeyMissingForVoiceInput
|
||||
? ExtL10n.string("keyboard.mic.disabled.missingApiKey")
|
||||
: ""
|
||||
applyAPIKeyAvailability(store: store, into: state)
|
||||
}
|
||||
|
||||
/// Cloud without ASR/LLM keys blocks the mic. Local ASR still works when
|
||||
/// the polish key is missing — show a soft tip above the mic instead.
|
||||
private func applyAPIKeyAvailability(
|
||||
store: AppGroupStore,
|
||||
into state: KeyboardViewController.State
|
||||
) {
|
||||
state.aiServiceAvailable = !store.isPolishKeyMissing
|
||||
if store.isCloudAPIKeyMissingForVoiceInput {
|
||||
state.micDisabled = true
|
||||
state.micDisabledHint = ExtL10n.string("keyboard.mic.disabled.missingApiKey")
|
||||
} else if store.isPolishKeyMissing {
|
||||
state.micDisabled = false
|
||||
state.micDisabledHint = ExtL10n.string("keyboard.mic.hint.missingPolishApiKey")
|
||||
} else {
|
||||
state.micDisabled = false
|
||||
state.micDisabledHint = ""
|
||||
}
|
||||
}
|
||||
|
||||
/// Persist `mode` to the App Group store.
|
||||
|
||||
@@ -54,9 +54,17 @@ final class KeyboardFlowCoordinator {
|
||||
private var editHostConfirmed = false
|
||||
private var cancelledEditUtteranceIDs: Set<UUID> = []
|
||||
private var cancelledDictationUtteranceIDs: Set<UUID> = []
|
||||
private var cancelledAIUtteranceIDs: Set<UUID> = []
|
||||
var onEditHostRecordingConfirmed: () -> Void = {}
|
||||
var onEditResult: (FlowResult) -> Void = { _ in }
|
||||
var onEditFailure: (String) -> Void = { _ in }
|
||||
var onAIUtterancePrepared: (UUID) -> Void = { _ in }
|
||||
var onAIRecordingStarted: (UUID) -> Void = { _ in }
|
||||
var onAIRecognitionStarted: (UUID) -> Void = { _ in }
|
||||
var onAITranscript: (String, UUID, FlowResult.Status) -> Void = { _, _, _ in }
|
||||
var onAIStreamingAnswer: (String, UUID) -> Void = { _, _ in }
|
||||
var onAIResult: (FlowResult) -> Void = { _ in }
|
||||
var onAIFailure: (String, UUID?) -> Void = { _, _ in }
|
||||
/// Utterance whose final result we already inserted (or failed). Prevents
|
||||
/// `adoptHostBusyStateIfNeeded` from re-entering `.processing` after a
|
||||
/// stale App Group snapshot still says `reason=processing`.
|
||||
@@ -178,6 +186,7 @@ final class KeyboardFlowCoordinator {
|
||||
adoptPendingResultIfNeeded()
|
||||
consumePendingFlowDeliveryIfNeeded()
|
||||
consumeEditStartFailureIfNeeded()
|
||||
consumeAIStartFailureIfNeeded()
|
||||
|
||||
recoverFromDeadHostIfNeeded()
|
||||
|
||||
@@ -243,6 +252,15 @@ final class KeyboardFlowCoordinator {
|
||||
completeEditResult(result, outcome: .rejected)
|
||||
}
|
||||
|
||||
private func consumeAIStartFailureIfNeeded() {
|
||||
guard currentUtteranceRequest?.isAIQuestion == true,
|
||||
let result = matchingResult(),
|
||||
isTerminalFailure(result) else {
|
||||
return
|
||||
}
|
||||
completeAIFailure(result)
|
||||
}
|
||||
|
||||
private func recomputeMicVoiceAvailability() {
|
||||
FlowSessionBridge.reloadFromDisk()
|
||||
let readySnapshot = FlowSessionBridge.readySnapshot()
|
||||
@@ -617,13 +635,76 @@ final class KeyboardFlowCoordinator {
|
||||
startUtterance(.editLastInput(reference))
|
||||
}
|
||||
|
||||
func beginAIRecording(
|
||||
conversationID: UUID
|
||||
) -> FlowUtteranceStartDisposition {
|
||||
startUtterance(.aiQuestion(conversationID: conversationID))
|
||||
}
|
||||
|
||||
func stopAIRecording() {
|
||||
guard currentUtteranceRequest?.isAIQuestion == true else { return }
|
||||
pressEnded()
|
||||
}
|
||||
|
||||
func cancelAIRecording() {
|
||||
guard currentUtteranceRequest?.isAIQuestion == true else { return }
|
||||
recordWhenHostReady = false
|
||||
recordAfterHandoff = false
|
||||
isPendingFlowStart = false
|
||||
flowStartDeadline = 0
|
||||
coldStartDebouncer.reset()
|
||||
stopHostReadyWait()
|
||||
stopFlowWatchdog()
|
||||
stopUtteranceCountdown()
|
||||
ExtensionScreenWakeLock.release()
|
||||
state.level = 0
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
|
||||
guard let utteranceID = currentUtteranceId,
|
||||
isFlowRecording || isAwaitingFlowResult else {
|
||||
clearUnissuedUtterance()
|
||||
isFlowRecording = false
|
||||
isAwaitingFlowResult = false
|
||||
recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
|
||||
cancelledAIUtteranceIDs.insert(utteranceID)
|
||||
writeCommand(.abort)
|
||||
isFlowRecording = false
|
||||
isAwaitingFlowResult = true
|
||||
startFlowResultWatchdog()
|
||||
recomputeMicVoiceAvailability()
|
||||
}
|
||||
|
||||
func endAIConversation(_ conversationID: UUID) {
|
||||
guard let sessionID = FlowSessionBridge.readySnapshot()?.sessionId
|
||||
?? activeSessionId else {
|
||||
return
|
||||
}
|
||||
FlowSessionBridge.writeCommand(
|
||||
FlowCommand(
|
||||
sessionId: sessionID,
|
||||
utteranceId: UUID(),
|
||||
commandSeq: nextCommandSeq(),
|
||||
action: .endAIConversation,
|
||||
localeId: state.localeId,
|
||||
aiConversationID: conversationID
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
func stopEditRecording() {
|
||||
guard currentUtteranceRequest?.isEdit == true else { return }
|
||||
pressEnded()
|
||||
}
|
||||
|
||||
func cancelCurrentDictation() {
|
||||
guard currentUtteranceRequest?.isEdit != true else { return }
|
||||
guard currentUtteranceRequest?.isEdit != true,
|
||||
currentUtteranceRequest?.isAIQuestion != true else {
|
||||
return
|
||||
}
|
||||
|
||||
recordWhenHostReady = false
|
||||
recordAfterHandoff = false
|
||||
@@ -733,6 +814,50 @@ final class KeyboardFlowCoordinator {
|
||||
resetEditTransportState()
|
||||
}
|
||||
|
||||
private func completeAIResult(_ result: FlowResult) {
|
||||
onAIResult(result)
|
||||
FlowSessionBridge.writeAck(
|
||||
FlowAck(
|
||||
sessionId: result.sessionId,
|
||||
utteranceId: result.utteranceId,
|
||||
commandSeq: result.commandSeq,
|
||||
hostGeneration: result.hostGeneration,
|
||||
revision: result.revision
|
||||
)
|
||||
)
|
||||
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
||||
lastConsumedUtteranceId = result.utteranceId
|
||||
lastStoppedUtteranceId = nil
|
||||
resetEditTransportState()
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
recomputeMicVoiceAvailability()
|
||||
}
|
||||
|
||||
private func completeAIFailure(_ result: FlowResult) {
|
||||
onAIFailure(
|
||||
result.text ?? ExtL10n.string("keyboard.ai.error.requestFailed"),
|
||||
result.utteranceId
|
||||
)
|
||||
FlowSessionBridge.writeAck(
|
||||
FlowAck(
|
||||
sessionId: result.sessionId,
|
||||
utteranceId: result.utteranceId,
|
||||
commandSeq: result.commandSeq,
|
||||
hostGeneration: result.hostGeneration,
|
||||
revision: result.revision,
|
||||
deliveryOutcome: .rejected
|
||||
)
|
||||
)
|
||||
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
||||
lastConsumedUtteranceId = result.utteranceId
|
||||
lastStoppedUtteranceId = nil
|
||||
resetEditTransportState()
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
recomputeMicVoiceAvailability()
|
||||
}
|
||||
|
||||
func pressBegan() {
|
||||
_ = startUtterance(.dictation)
|
||||
}
|
||||
@@ -762,6 +887,9 @@ final class KeyboardFlowCoordinator {
|
||||
currentUtteranceId = utteranceID
|
||||
currentUtteranceRequest = request
|
||||
editHostConfirmed = false
|
||||
if request.isAIQuestion {
|
||||
onAIUtterancePrepared(utteranceID)
|
||||
}
|
||||
currentStartDeadlineAt = Date().timeIntervalSince1970
|
||||
+ FlowSessionKeys.utteranceStartBudget
|
||||
|
||||
@@ -863,7 +991,10 @@ final class KeyboardFlowCoordinator {
|
||||
writeCommand(.stopRecording)
|
||||
debug("pressEnded wrote stop command")
|
||||
state.phase = .processing
|
||||
if currentUtteranceRequest?.isEdit != true {
|
||||
if currentUtteranceRequest?.isAIQuestion == true,
|
||||
let currentUtteranceId {
|
||||
onAIRecognitionStarted(currentUtteranceId)
|
||||
} else if currentUtteranceRequest?.isEdit != true {
|
||||
state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
|
||||
}
|
||||
startFlowResultWatchdog()
|
||||
@@ -923,6 +1054,16 @@ final class KeyboardFlowCoordinator {
|
||||
state.lastTranscript = ""
|
||||
return
|
||||
}
|
||||
if currentUtteranceRequest?.isAIQuestion == true {
|
||||
onAIFailure(
|
||||
ExtL10n.string("keyboard.error.manualOpenForFlow"),
|
||||
currentUtteranceId
|
||||
)
|
||||
resetEditTransportState()
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
return
|
||||
}
|
||||
showManualOpenHint(path: "startflow")
|
||||
recomputeMicVoiceAvailability()
|
||||
return
|
||||
@@ -972,13 +1113,16 @@ final class KeyboardFlowCoordinator {
|
||||
commandSeq: nextCommandSeq(),
|
||||
action: action,
|
||||
localeId: state.localeId,
|
||||
fieldContext: action == .stopRecording ? fieldContextProvider() : nil,
|
||||
fieldContext: action == .stopRecording && !request.isAIQuestion
|
||||
? fieldContextProvider()
|
||||
: nil,
|
||||
utteranceMode: mode,
|
||||
editSourceText: action == .startRecording
|
||||
? request.editSourceText
|
||||
: nil,
|
||||
sourceHistoryEntryID: request.sourceHistoryEntryID,
|
||||
sourceHistoryEntryRevision: request.sourceHistoryEntryRevision,
|
||||
aiConversationID: request.aiConversationID,
|
||||
startDeadlineAt: action == .startRecording ? currentStartDeadlineAt : nil,
|
||||
processingDeadlineAt: action == .stopRecording && request.isEdit
|
||||
? Date().timeIntervalSince1970
|
||||
@@ -1027,6 +1171,10 @@ final class KeyboardFlowCoordinator {
|
||||
consumeCancelledEditResultIfNeeded(result) {
|
||||
return
|
||||
}
|
||||
if let result = matchingResult(),
|
||||
consumeCancelledAIResultIfNeeded(result) {
|
||||
return
|
||||
}
|
||||
if let result = matchingResult(), result.status == .final, let text = result.text, !text.isEmpty {
|
||||
isAwaitingFlowResult = false
|
||||
stopFlowWatchdog()
|
||||
@@ -1035,6 +1183,10 @@ final class KeyboardFlowCoordinator {
|
||||
onEditResult(result)
|
||||
return
|
||||
}
|
||||
if result.resolvedUtteranceMode == .aiQuestion {
|
||||
completeAIResult(result)
|
||||
return
|
||||
}
|
||||
textInserter.handleFlowTranscript(
|
||||
TranscriptionDelivery(
|
||||
text: text,
|
||||
@@ -1083,6 +1235,10 @@ final class KeyboardFlowCoordinator {
|
||||
state.phase = .idle
|
||||
return
|
||||
}
|
||||
if result.resolvedUtteranceMode == .aiQuestion {
|
||||
completeAIFailure(result)
|
||||
return
|
||||
}
|
||||
FlowSessionBridge.writeAck(
|
||||
FlowAck(
|
||||
sessionId: result.sessionId,
|
||||
@@ -1172,6 +1328,30 @@ final class KeyboardFlowCoordinator {
|
||||
return true
|
||||
}
|
||||
|
||||
private func consumeCancelledAIResultIfNeeded(_ result: FlowResult) -> Bool {
|
||||
guard cancelledAIUtteranceIDs.contains(result.utteranceId),
|
||||
result.status == .final || isTerminalFailure(result) else {
|
||||
return false
|
||||
}
|
||||
FlowSessionBridge.writeAck(
|
||||
FlowAck(
|
||||
sessionId: result.sessionId,
|
||||
utteranceId: result.utteranceId,
|
||||
commandSeq: result.commandSeq,
|
||||
hostGeneration: result.hostGeneration,
|
||||
revision: result.revision,
|
||||
deliveryOutcome: .rejected
|
||||
)
|
||||
)
|
||||
cancelledAIUtteranceIDs.remove(result.utteranceId)
|
||||
lastConsumedUtteranceId = result.utteranceId
|
||||
resetEditTransportState()
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
recomputeMicVoiceAvailability()
|
||||
return true
|
||||
}
|
||||
|
||||
private func adoptPendingResultIfNeeded() {
|
||||
guard !isAwaitingFlowResult, currentUtteranceId == nil,
|
||||
let pendingId = FlowSessionBridge.pendingKeyboardUtteranceId(),
|
||||
@@ -1343,6 +1523,18 @@ final class KeyboardFlowCoordinator {
|
||||
)
|
||||
return
|
||||
}
|
||||
if let id = currentUtteranceId,
|
||||
cancelledAIUtteranceIDs.remove(id) != nil {
|
||||
stopUtteranceCountdown()
|
||||
stopFlowWatchdog()
|
||||
ExtensionScreenWakeLock.release()
|
||||
resetEditTransportState()
|
||||
state.level = 0
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
if deliverRawFallbackIfAvailable(reason: "hostDisconnected") {
|
||||
return
|
||||
}
|
||||
@@ -1367,6 +1559,14 @@ final class KeyboardFlowCoordinator {
|
||||
recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
if currentUtteranceRequest?.isAIQuestion == true {
|
||||
onAIFailure(message, currentUtteranceId)
|
||||
resetEditTransportState()
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
state.phase = .error(.flowSessionExpired, message: message)
|
||||
scheduleAutoClearError()
|
||||
recomputeMicVoiceAvailability()
|
||||
@@ -1469,6 +1669,12 @@ final class KeyboardFlowCoordinator {
|
||||
if currentUtteranceRequest?.isEdit == true {
|
||||
onEditFailure(ExtL10n.string("keyboard.edit.error.startTimeout"))
|
||||
abortEditRecording()
|
||||
} else if currentUtteranceRequest?.isAIQuestion == true {
|
||||
onAIFailure(
|
||||
ExtL10n.string("keyboard.ai.error.startTimeout"),
|
||||
currentUtteranceId
|
||||
)
|
||||
cancelAIRecording()
|
||||
} else {
|
||||
state.phase = .error(
|
||||
.hostAudioUnavailable,
|
||||
@@ -1538,6 +1744,10 @@ final class KeyboardFlowCoordinator {
|
||||
state.phase = currentUtteranceRequest?.isEdit == true
|
||||
? .requestingPermissions
|
||||
: .recording
|
||||
if currentUtteranceRequest?.isAIQuestion == true,
|
||||
let currentUtteranceId {
|
||||
onAIRecordingStarted(currentUtteranceId)
|
||||
}
|
||||
recomputeMicVoiceAvailability()
|
||||
if let view = wakeLockView() {
|
||||
ExtensionScreenWakeLock.acquire(from: view)
|
||||
@@ -1658,11 +1868,19 @@ final class KeyboardFlowCoordinator {
|
||||
guard isFlowRecording || isAwaitingFlowResult else { return }
|
||||
switch state.phase {
|
||||
case .recording, .processing:
|
||||
if let result = matchingResult(),
|
||||
result.status == .partial || result.status == .rawReady,
|
||||
guard let result = matchingResult() else { return }
|
||||
if result.status == .streaming,
|
||||
result.resolvedUtteranceMode == .aiQuestion {
|
||||
onAIStreamingAnswer(result.text ?? "", result.utteranceId)
|
||||
return
|
||||
}
|
||||
if (result.status == .partial || result.status == .rawReady),
|
||||
let partial = result.text,
|
||||
!partial.isEmpty {
|
||||
state.lastTranscript = partial
|
||||
if result.resolvedUtteranceMode == .aiQuestion {
|
||||
onAITranscript(partial, result.utteranceId, result.status)
|
||||
}
|
||||
}
|
||||
default:
|
||||
break
|
||||
@@ -1679,11 +1897,21 @@ final class KeyboardFlowCoordinator {
|
||||
let isCancelledDictation = currentUtteranceId.map {
|
||||
cancelledDictationUtteranceIDs.contains($0)
|
||||
} ?? false
|
||||
let resultTimeout = isCancelledEdit || isCancelledDictation
|
||||
? FlowSessionKeys.utteranceStartBudget
|
||||
: (currentUtteranceRequest?.isEdit != true
|
||||
? FlowWatchdog.resultTimeout(engineMode: state.engineMode)
|
||||
: FlowSessionKeys.editLastInputProcessingBudget)
|
||||
let isCancelledAI = currentUtteranceId.map {
|
||||
cancelledAIUtteranceIDs.contains($0)
|
||||
} ?? false
|
||||
let resultTimeout: TimeInterval
|
||||
if isCancelledEdit || isCancelledDictation || isCancelledAI {
|
||||
resultTimeout = FlowSessionKeys.utteranceStartBudget
|
||||
} else if currentUtteranceRequest?.isAIQuestion == true {
|
||||
resultTimeout = FlowSessionKeys.keyboardAIResultTimeout(
|
||||
engineMode: state.engineMode
|
||||
)
|
||||
} else if currentUtteranceRequest?.isEdit == true {
|
||||
resultTimeout = FlowSessionKeys.editLastInputProcessingBudget
|
||||
} else {
|
||||
resultTimeout = FlowWatchdog.resultTimeout(engineMode: state.engineMode)
|
||||
}
|
||||
debug("resultWatchdog started timeout=\(Int(resultTimeout))s engine=\(state.engineMode)")
|
||||
flowWatchdogTask = Task { @MainActor [weak self] in
|
||||
while let self, !Task.isCancelled {
|
||||
@@ -1700,6 +1928,10 @@ final class KeyboardFlowCoordinator {
|
||||
self.consumeCancelledEditResultIfNeeded(result) {
|
||||
return
|
||||
}
|
||||
if let result = self.matchingResult(),
|
||||
self.consumeCancelledAIResultIfNeeded(result) {
|
||||
return
|
||||
}
|
||||
if let result = self.matchingResult(), result.status == .final, let text = result.text, !text.isEmpty {
|
||||
self.isAwaitingFlowResult = false
|
||||
self.stopFlowWatchdog()
|
||||
@@ -1707,6 +1939,10 @@ final class KeyboardFlowCoordinator {
|
||||
self.onEditResult(result)
|
||||
return
|
||||
}
|
||||
if result.resolvedUtteranceMode == .aiQuestion {
|
||||
self.completeAIResult(result)
|
||||
return
|
||||
}
|
||||
self.textInserter.handleFlowTranscript(
|
||||
TranscriptionDelivery(
|
||||
text: text,
|
||||
@@ -1749,6 +1985,10 @@ final class KeyboardFlowCoordinator {
|
||||
self.state.phase = .idle
|
||||
return
|
||||
}
|
||||
if result.resolvedUtteranceMode == .aiQuestion {
|
||||
self.completeAIFailure(result)
|
||||
return
|
||||
}
|
||||
FlowSessionBridge.writeAck(
|
||||
FlowAck(
|
||||
sessionId: result.sessionId,
|
||||
@@ -1823,6 +2063,14 @@ final class KeyboardFlowCoordinator {
|
||||
self.recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
if let id = self.currentUtteranceId,
|
||||
self.cancelledAIUtteranceIDs.remove(id) != nil {
|
||||
self.resetEditTransportState()
|
||||
self.state.phase = .idle
|
||||
self.state.lastTranscript = ""
|
||||
self.recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
if self.currentUtteranceRequest?.isEdit == true {
|
||||
self.isAwaitingFlowResult = false
|
||||
self.stopFlowWatchdog()
|
||||
@@ -1837,6 +2085,21 @@ final class KeyboardFlowCoordinator {
|
||||
self.state.phase = .idle
|
||||
return
|
||||
}
|
||||
if self.currentUtteranceRequest?.isAIQuestion == true {
|
||||
let utteranceID = self.currentUtteranceId
|
||||
self.isAwaitingFlowResult = false
|
||||
self.stopFlowWatchdog()
|
||||
self.writeCommand(.abort)
|
||||
self.onAIFailure(
|
||||
ExtL10n.string("keyboard.ai.error.requestTimeout"),
|
||||
utteranceID
|
||||
)
|
||||
self.resetEditTransportState()
|
||||
self.state.phase = .idle
|
||||
self.state.lastTranscript = ""
|
||||
self.recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
if self.deliverRawFallbackIfAvailable(reason: "resultTimeout") {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -98,6 +98,42 @@ final class KeyboardTextInserter {
|
||||
OSGLog.keyboardExt.info("flow insert length=\(trimmed.count, privacy: .public)")
|
||||
}
|
||||
|
||||
/// Insert one explicitly confirmed AI answer and enqueue history/statistics
|
||||
/// only after the field mutation has been issued.
|
||||
@discardableResult
|
||||
func insertAIAnswer(_ answer: AIAnswer) -> Bool {
|
||||
let trimmed = answer.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return false }
|
||||
|
||||
let separator = DictationTextComposer.insertionSeparator(
|
||||
previousContext: contextBeforeInput(),
|
||||
insertion: trimmed
|
||||
)
|
||||
let inserted = separator + trimmed
|
||||
insertText(inserted)
|
||||
|
||||
let mutation = HistoryMutation(
|
||||
action: .append,
|
||||
entryID: answer.id,
|
||||
text: trimmed,
|
||||
engineMode: state.engineMode,
|
||||
source: .ai,
|
||||
usageCategory: .ai
|
||||
)
|
||||
HistoryMutationOutbox.enqueue(mutation)
|
||||
recordLastInsertion(
|
||||
inserted,
|
||||
displayText: trimmed,
|
||||
historyEntryID: answer.id,
|
||||
historyEntryRevision: 0,
|
||||
pendingHistoryMutationID: mutation.id
|
||||
)
|
||||
state.lastTranscript = ""
|
||||
state.level = 0
|
||||
OSGLog.keyboardExt.info("AI answer insert length=\(trimmed.count, privacy: .public)")
|
||||
return true
|
||||
}
|
||||
|
||||
/// Roll back the last voice insertion when it is still at the caret.
|
||||
func undoLastInsertion() {
|
||||
if undoLastEditIfPossible() {
|
||||
|
||||
Reference in New Issue
Block a user