feat(keyboard): ship AI mode surface with streaming search answers

Add the AI keyboard tab, Agent settings, and user-owned LLM key path for 1.7.0, including streaming answers and web-search transports without the built-in DeepSeek fallback.
This commit is contained in:
Rocky
2026-08-11 01:06:27 +08:00
parent f6212dcd2c
commit 3de665d254
81 changed files with 4467 additions and 398 deletions
+70 -3
View File
@@ -65,6 +65,7 @@ public final class KeyboardViewController: UIInputViewController {
private var textInserter: KeyboardTextInserter!
private var flowCoordinator: KeyboardFlowCoordinator!
private var lastInputEditCoordinator: LastInputEditCoordinator!
private var aiKeyboardCoordinator: AIKeyboardCoordinator!
private var configSync: KeyboardConfigSync!
/// UIKit may synchronously lay out the view during `viewDidLoad`.
/// Keep this optional so an early layout pass is harmless.
@@ -178,6 +179,12 @@ public final class KeyboardViewController: UIInputViewController {
TypingInputConfiguration.persistLastSurface(
preserve ? .voice : state.surface
)
if state.surface == .ai {
// AI context never survives a keyboard presentation, but the
// selected surface itself is restored on the next open.
TypingInputConfiguration.persistLastSurface(.ai)
aiKeyboardCoordinator.leave()
}
if !preserve {
prepareSurfaceForNextPresentation()
}
@@ -375,6 +382,44 @@ public final class KeyboardViewController: UIInputViewController {
flowCoordinator.onEditFailure = { [weak self] message in
self?.lastInputEditCoordinator.fail(message)
}
aiKeyboardCoordinator = AIKeyboardCoordinator(
state: state,
flow: flowCoordinator,
insertAnswer: { [weak self] answer in
self?.textInserter.insertAIAnswer(answer) ?? false
},
performReturn: { [weak self] in
self?.textDocumentProxy.insertText("\n")
}
)
flowCoordinator.onAIUtterancePrepared = { [weak self] utteranceID in
self?.aiKeyboardCoordinator.utterancePrepared(utteranceID)
}
flowCoordinator.onAIRecordingStarted = { [weak self] utteranceID in
self?.aiKeyboardCoordinator.recordingStarted(utteranceID)
}
flowCoordinator.onAIRecognitionStarted = { [weak self] utteranceID in
self?.aiKeyboardCoordinator.recognitionStarted(utteranceID)
}
flowCoordinator.onAITranscript = { [weak self] transcript, utteranceID, status in
self?.aiKeyboardCoordinator.receiveTranscript(
transcript,
utteranceID: utteranceID,
status: status
)
}
flowCoordinator.onAIStreamingAnswer = { [weak self] draft, utteranceID in
self?.aiKeyboardCoordinator.receivePartialAnswer(
draft,
utteranceID: utteranceID
)
}
flowCoordinator.onAIResult = { [weak self] result in
self?.aiKeyboardCoordinator.receive(result: result)
}
flowCoordinator.onAIFailure = { [weak self] message, utteranceID in
self?.aiKeyboardCoordinator.fail(message, utteranceID: utteranceID)
}
_ = textInserter.recoverPendingEditTransactionIfNeeded()
cursorDrag = CursorDragController(
@@ -409,6 +454,15 @@ public final class KeyboardViewController: UIInputViewController {
state.closeEditMode = { [weak self] in
self?.lastInputEditCoordinator.close()
}
state.tapAIMic = { [weak self] in
self?.aiKeyboardCoordinator.toggleMicrophone()
}
state.cancelAIInput = { [weak self] in
self?.aiKeyboardCoordinator.cancel()
}
state.sendAIAnswer = { [weak self] in
self?.aiKeyboardCoordinator.sendLatestAnswer()
}
state.openSettings = { [weak self] in self?.openHostApp() }
state.openInputMethodSetup = { [weak self] in self?.openHostApp(path: "deployrime") }
// The globe UIButton registers this controller's standard
@@ -471,6 +525,9 @@ public final class KeyboardViewController: UIInputViewController {
return
}
guard state.surface != surface else {
if surface == .ai {
aiKeyboardCoordinator.enterIfNeeded()
}
refreshKeyboardHeight()
return
}
@@ -478,11 +535,18 @@ public final class KeyboardViewController: UIInputViewController {
"applySurface \(state.surface.rawValue)\(surface.rawValue) \(OSGDiag.memoryTag())",
category: "boot"
)
let previousSurface = state.surface
if previousSurface == .ai, surface != .ai {
aiKeyboardCoordinator.leave()
}
state.surface = surface
if surface == .voice {
typingSession.leaveTypingMode()
} else {
if surface == .typing {
typingSession.enterTypingMode()
} else {
typingSession.leaveTypingMode()
}
if surface == .ai {
aiKeyboardCoordinator.enterIfNeeded()
}
refreshKeyboardHeight()
}
@@ -500,6 +564,9 @@ public final class KeyboardViewController: UIInputViewController {
category: "boot"
)
applySurface(resolved)
if resolved == .ai {
aiKeyboardCoordinator.beginNewPresentation()
}
}
/// When not remembering, snap to the static open preference while hidden
@@ -0,0 +1,162 @@
// AIKeyboardCoordinator.swift
// OSGKeyboard · Keyboard Extension
//
// Owns the temporary AI-mode UI state. Audio, ASR, and LLM work remain in the
// shared Flow transport and host app; this coordinator never performs network
// work and never inserts an answer before explicit user confirmation.
import Foundation
import OSGKeyboardShared
@MainActor
final class AIKeyboardCoordinator {
private let state: KeyboardState
private let flow: KeyboardFlowCoordinator
private let insertAnswer: (AIAnswer) -> Bool
private let performReturn: () -> Void
init(
state: KeyboardState,
flow: KeyboardFlowCoordinator,
insertAnswer: @escaping (AIAnswer) -> Bool,
performReturn: @escaping () -> Void
) {
self.state = state
self.flow = flow
self.insertAnswer = insertAnswer
self.performReturn = performReturn
}
func beginNewPresentation() {
endConversationIfNeeded()
state.aiSession.enter()
}
func enterIfNeeded() {
guard !state.aiSession.isActive else { return }
state.aiSession.enter()
}
func leave() {
if state.aiSession.isBusy {
flow.cancelAIRecording()
}
endConversationIfNeeded()
state.aiSession.leave()
}
func toggleMicrophone() {
switch state.aiSession.phase {
case .listening:
guard let utteranceID = state.aiSession.activeUtteranceID else { return }
state.aiSession.beginRecognizing(utteranceID: utteranceID)
flow.stopAIRecording()
case .idle, .ready, .awaitingSend, .inserted, .sent, .failed:
enterIfNeeded()
guard let conversationID = state.aiSession.conversationID else { return }
let disposition = flow.beginAIRecording(conversationID: conversationID)
if case .rejected(let rejection) = disposition {
state.aiSession.fail(message(for: rejection), utteranceID: nil)
}
case .inactive:
enterIfNeeded()
toggleMicrophone()
case .preparing, .recognizing, .generating:
break
}
}
func cancel() {
guard state.aiSession.isBusy else { return }
flow.cancelAIRecording()
state.aiSession.cancelCurrentWork()
}
func sendLatestAnswer() {
if state.aiSession.canInsert, let answer = state.aiSession.answer {
guard insertAnswer(answer) else { return }
state.aiSession.markAnswerInserted(
offersSend: state.returnKeyRole == .send
)
} else if state.aiSession.canSend {
state.aiSession.markAnswerSent()
// Let the host consume the inserted answer before issuing Return.
Task { @MainActor [weak self] in
await Task.yield()
self?.performReturn()
}
}
}
func utterancePrepared(_ utteranceID: UUID) {
state.aiSession.beginPreparing(utteranceID: utteranceID)
}
func recordingStarted(_ utteranceID: UUID) {
state.aiSession.beginListening(utteranceID: utteranceID)
}
func recognitionStarted(_ utteranceID: UUID) {
state.aiSession.beginRecognizing(utteranceID: utteranceID)
}
func receiveTranscript(
_ transcript: String,
utteranceID: UUID,
status: FlowResult.Status
) {
state.aiSession.updateTranscript(transcript, utteranceID: utteranceID)
if status == .rawReady {
state.aiSession.beginGenerating(
question: transcript,
utteranceID: utteranceID
)
}
}
func receivePartialAnswer(_ draft: String, utteranceID: UUID) {
state.aiSession.receivePartialAnswer(draft, utteranceID: utteranceID)
}
func receive(result: FlowResult) {
guard result.resolvedUtteranceMode == .aiQuestion else {
return
}
guard result.aiConversationID == state.aiSession.conversationID,
let answer = result.text,
!answer.isEmpty else {
state.aiSession.fail(
ExtL10n.string("keyboard.ai.error.requestFailed"),
utteranceID: result.utteranceId
)
return
}
state.aiSession.receiveAnswer(answer, utteranceID: result.utteranceId)
}
func fail(_ message: String, utteranceID: UUID?) {
state.aiSession.fail(message, utteranceID: utteranceID)
}
private func endConversationIfNeeded() {
guard let conversationID = state.aiSession.conversationID else { return }
flow.endAIConversation(conversationID)
}
private func message(for rejection: FlowUtteranceStartRejection) -> String {
switch rejection {
case .onboardingIncomplete:
return ExtL10n.string("keyboard.hint.finishSetupInApp")
case .missingAPIKey:
return ExtL10n.string("keyboard.ai.error.missingAPIKey")
case .noFullAccess:
return ExtL10n.string("keyboard.error.fullAccessRequired")
case .appGroupUnavailable:
return ExtL10n.string("keyboard.error.appGroupCommunication")
case .hostUnavailable:
return ExtL10n.string("keyboard.flow.hostDisconnected")
case .pipelineBusy:
return ExtL10n.string("keyboard.ai.error.pipelineBusy")
}
}
}
@@ -42,10 +42,7 @@ public struct AppGroupPersistor {
state.handednessPreference = store.handednessPreference
state.cursorDragNavigationEnabled = store.cursorDragNavigationEnabled
state.keyboardHapticIntensity = store.keyboardHapticIntensity
state.micDisabled = store.isCloudAPIKeyMissingForVoiceInput
state.micDisabledHint = store.isCloudAPIKeyMissingForVoiceInput
? ExtL10n.string("keyboard.mic.disabled.missingApiKey")
: ""
applyAPIKeyAvailability(store: store, into: state)
#if DEBUG
// Print a masked view of the live App Group config so we can see
@@ -95,10 +92,26 @@ public struct AppGroupPersistor {
state.handednessPreference = store.handednessPreference
state.cursorDragNavigationEnabled = store.cursorDragNavigationEnabled
state.keyboardHapticIntensity = store.keyboardHapticIntensity
state.micDisabled = store.isCloudAPIKeyMissingForVoiceInput
state.micDisabledHint = store.isCloudAPIKeyMissingForVoiceInput
? ExtL10n.string("keyboard.mic.disabled.missingApiKey")
: ""
applyAPIKeyAvailability(store: store, into: state)
}
/// Cloud without ASR/LLM keys blocks the mic. Local ASR still works when
/// the polish key is missing show a soft tip above the mic instead.
private func applyAPIKeyAvailability(
store: AppGroupStore,
into state: KeyboardViewController.State
) {
state.aiServiceAvailable = !store.isPolishKeyMissing
if store.isCloudAPIKeyMissingForVoiceInput {
state.micDisabled = true
state.micDisabledHint = ExtL10n.string("keyboard.mic.disabled.missingApiKey")
} else if store.isPolishKeyMissing {
state.micDisabled = false
state.micDisabledHint = ExtL10n.string("keyboard.mic.hint.missingPolishApiKey")
} else {
state.micDisabled = false
state.micDisabledHint = ""
}
}
/// Persist `mode` to the App Group store.
@@ -54,9 +54,17 @@ final class KeyboardFlowCoordinator {
private var editHostConfirmed = false
private var cancelledEditUtteranceIDs: Set<UUID> = []
private var cancelledDictationUtteranceIDs: Set<UUID> = []
private var cancelledAIUtteranceIDs: Set<UUID> = []
var onEditHostRecordingConfirmed: () -> Void = {}
var onEditResult: (FlowResult) -> Void = { _ in }
var onEditFailure: (String) -> Void = { _ in }
var onAIUtterancePrepared: (UUID) -> Void = { _ in }
var onAIRecordingStarted: (UUID) -> Void = { _ in }
var onAIRecognitionStarted: (UUID) -> Void = { _ in }
var onAITranscript: (String, UUID, FlowResult.Status) -> Void = { _, _, _ in }
var onAIStreamingAnswer: (String, UUID) -> Void = { _, _ in }
var onAIResult: (FlowResult) -> Void = { _ in }
var onAIFailure: (String, UUID?) -> Void = { _, _ in }
/// Utterance whose final result we already inserted (or failed). Prevents
/// `adoptHostBusyStateIfNeeded` from re-entering `.processing` after a
/// stale App Group snapshot still says `reason=processing`.
@@ -178,6 +186,7 @@ final class KeyboardFlowCoordinator {
adoptPendingResultIfNeeded()
consumePendingFlowDeliveryIfNeeded()
consumeEditStartFailureIfNeeded()
consumeAIStartFailureIfNeeded()
recoverFromDeadHostIfNeeded()
@@ -243,6 +252,15 @@ final class KeyboardFlowCoordinator {
completeEditResult(result, outcome: .rejected)
}
private func consumeAIStartFailureIfNeeded() {
guard currentUtteranceRequest?.isAIQuestion == true,
let result = matchingResult(),
isTerminalFailure(result) else {
return
}
completeAIFailure(result)
}
private func recomputeMicVoiceAvailability() {
FlowSessionBridge.reloadFromDisk()
let readySnapshot = FlowSessionBridge.readySnapshot()
@@ -617,13 +635,76 @@ final class KeyboardFlowCoordinator {
startUtterance(.editLastInput(reference))
}
func beginAIRecording(
conversationID: UUID
) -> FlowUtteranceStartDisposition {
startUtterance(.aiQuestion(conversationID: conversationID))
}
func stopAIRecording() {
guard currentUtteranceRequest?.isAIQuestion == true else { return }
pressEnded()
}
func cancelAIRecording() {
guard currentUtteranceRequest?.isAIQuestion == true else { return }
recordWhenHostReady = false
recordAfterHandoff = false
isPendingFlowStart = false
flowStartDeadline = 0
coldStartDebouncer.reset()
stopHostReadyWait()
stopFlowWatchdog()
stopUtteranceCountdown()
ExtensionScreenWakeLock.release()
state.level = 0
state.phase = .idle
state.lastTranscript = ""
guard let utteranceID = currentUtteranceId,
isFlowRecording || isAwaitingFlowResult else {
clearUnissuedUtterance()
isFlowRecording = false
isAwaitingFlowResult = false
recomputeMicVoiceAvailability()
return
}
cancelledAIUtteranceIDs.insert(utteranceID)
writeCommand(.abort)
isFlowRecording = false
isAwaitingFlowResult = true
startFlowResultWatchdog()
recomputeMicVoiceAvailability()
}
func endAIConversation(_ conversationID: UUID) {
guard let sessionID = FlowSessionBridge.readySnapshot()?.sessionId
?? activeSessionId else {
return
}
FlowSessionBridge.writeCommand(
FlowCommand(
sessionId: sessionID,
utteranceId: UUID(),
commandSeq: nextCommandSeq(),
action: .endAIConversation,
localeId: state.localeId,
aiConversationID: conversationID
)
)
}
func stopEditRecording() {
guard currentUtteranceRequest?.isEdit == true else { return }
pressEnded()
}
func cancelCurrentDictation() {
guard currentUtteranceRequest?.isEdit != true else { return }
guard currentUtteranceRequest?.isEdit != true,
currentUtteranceRequest?.isAIQuestion != true else {
return
}
recordWhenHostReady = false
recordAfterHandoff = false
@@ -733,6 +814,50 @@ final class KeyboardFlowCoordinator {
resetEditTransportState()
}
private func completeAIResult(_ result: FlowResult) {
onAIResult(result)
FlowSessionBridge.writeAck(
FlowAck(
sessionId: result.sessionId,
utteranceId: result.utteranceId,
commandSeq: result.commandSeq,
hostGeneration: result.hostGeneration,
revision: result.revision
)
)
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
lastConsumedUtteranceId = result.utteranceId
lastStoppedUtteranceId = nil
resetEditTransportState()
state.phase = .idle
state.lastTranscript = ""
recomputeMicVoiceAvailability()
}
private func completeAIFailure(_ result: FlowResult) {
onAIFailure(
result.text ?? ExtL10n.string("keyboard.ai.error.requestFailed"),
result.utteranceId
)
FlowSessionBridge.writeAck(
FlowAck(
sessionId: result.sessionId,
utteranceId: result.utteranceId,
commandSeq: result.commandSeq,
hostGeneration: result.hostGeneration,
revision: result.revision,
deliveryOutcome: .rejected
)
)
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
lastConsumedUtteranceId = result.utteranceId
lastStoppedUtteranceId = nil
resetEditTransportState()
state.phase = .idle
state.lastTranscript = ""
recomputeMicVoiceAvailability()
}
func pressBegan() {
_ = startUtterance(.dictation)
}
@@ -762,6 +887,9 @@ final class KeyboardFlowCoordinator {
currentUtteranceId = utteranceID
currentUtteranceRequest = request
editHostConfirmed = false
if request.isAIQuestion {
onAIUtterancePrepared(utteranceID)
}
currentStartDeadlineAt = Date().timeIntervalSince1970
+ FlowSessionKeys.utteranceStartBudget
@@ -863,7 +991,10 @@ final class KeyboardFlowCoordinator {
writeCommand(.stopRecording)
debug("pressEnded wrote stop command")
state.phase = .processing
if currentUtteranceRequest?.isEdit != true {
if currentUtteranceRequest?.isAIQuestion == true,
let currentUtteranceId {
onAIRecognitionStarted(currentUtteranceId)
} else if currentUtteranceRequest?.isEdit != true {
state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
}
startFlowResultWatchdog()
@@ -923,6 +1054,16 @@ final class KeyboardFlowCoordinator {
state.lastTranscript = ""
return
}
if currentUtteranceRequest?.isAIQuestion == true {
onAIFailure(
ExtL10n.string("keyboard.error.manualOpenForFlow"),
currentUtteranceId
)
resetEditTransportState()
state.phase = .idle
state.lastTranscript = ""
return
}
showManualOpenHint(path: "startflow")
recomputeMicVoiceAvailability()
return
@@ -972,13 +1113,16 @@ final class KeyboardFlowCoordinator {
commandSeq: nextCommandSeq(),
action: action,
localeId: state.localeId,
fieldContext: action == .stopRecording ? fieldContextProvider() : nil,
fieldContext: action == .stopRecording && !request.isAIQuestion
? fieldContextProvider()
: nil,
utteranceMode: mode,
editSourceText: action == .startRecording
? request.editSourceText
: nil,
sourceHistoryEntryID: request.sourceHistoryEntryID,
sourceHistoryEntryRevision: request.sourceHistoryEntryRevision,
aiConversationID: request.aiConversationID,
startDeadlineAt: action == .startRecording ? currentStartDeadlineAt : nil,
processingDeadlineAt: action == .stopRecording && request.isEdit
? Date().timeIntervalSince1970
@@ -1027,6 +1171,10 @@ final class KeyboardFlowCoordinator {
consumeCancelledEditResultIfNeeded(result) {
return
}
if let result = matchingResult(),
consumeCancelledAIResultIfNeeded(result) {
return
}
if let result = matchingResult(), result.status == .final, let text = result.text, !text.isEmpty {
isAwaitingFlowResult = false
stopFlowWatchdog()
@@ -1035,6 +1183,10 @@ final class KeyboardFlowCoordinator {
onEditResult(result)
return
}
if result.resolvedUtteranceMode == .aiQuestion {
completeAIResult(result)
return
}
textInserter.handleFlowTranscript(
TranscriptionDelivery(
text: text,
@@ -1083,6 +1235,10 @@ final class KeyboardFlowCoordinator {
state.phase = .idle
return
}
if result.resolvedUtteranceMode == .aiQuestion {
completeAIFailure(result)
return
}
FlowSessionBridge.writeAck(
FlowAck(
sessionId: result.sessionId,
@@ -1172,6 +1328,30 @@ final class KeyboardFlowCoordinator {
return true
}
private func consumeCancelledAIResultIfNeeded(_ result: FlowResult) -> Bool {
guard cancelledAIUtteranceIDs.contains(result.utteranceId),
result.status == .final || isTerminalFailure(result) else {
return false
}
FlowSessionBridge.writeAck(
FlowAck(
sessionId: result.sessionId,
utteranceId: result.utteranceId,
commandSeq: result.commandSeq,
hostGeneration: result.hostGeneration,
revision: result.revision,
deliveryOutcome: .rejected
)
)
cancelledAIUtteranceIDs.remove(result.utteranceId)
lastConsumedUtteranceId = result.utteranceId
resetEditTransportState()
state.phase = .idle
state.lastTranscript = ""
recomputeMicVoiceAvailability()
return true
}
private func adoptPendingResultIfNeeded() {
guard !isAwaitingFlowResult, currentUtteranceId == nil,
let pendingId = FlowSessionBridge.pendingKeyboardUtteranceId(),
@@ -1343,6 +1523,18 @@ final class KeyboardFlowCoordinator {
)
return
}
if let id = currentUtteranceId,
cancelledAIUtteranceIDs.remove(id) != nil {
stopUtteranceCountdown()
stopFlowWatchdog()
ExtensionScreenWakeLock.release()
resetEditTransportState()
state.level = 0
state.phase = .idle
state.lastTranscript = ""
recomputeMicVoiceAvailability()
return
}
if deliverRawFallbackIfAvailable(reason: "hostDisconnected") {
return
}
@@ -1367,6 +1559,14 @@ final class KeyboardFlowCoordinator {
recomputeMicVoiceAvailability()
return
}
if currentUtteranceRequest?.isAIQuestion == true {
onAIFailure(message, currentUtteranceId)
resetEditTransportState()
state.phase = .idle
state.lastTranscript = ""
recomputeMicVoiceAvailability()
return
}
state.phase = .error(.flowSessionExpired, message: message)
scheduleAutoClearError()
recomputeMicVoiceAvailability()
@@ -1469,6 +1669,12 @@ final class KeyboardFlowCoordinator {
if currentUtteranceRequest?.isEdit == true {
onEditFailure(ExtL10n.string("keyboard.edit.error.startTimeout"))
abortEditRecording()
} else if currentUtteranceRequest?.isAIQuestion == true {
onAIFailure(
ExtL10n.string("keyboard.ai.error.startTimeout"),
currentUtteranceId
)
cancelAIRecording()
} else {
state.phase = .error(
.hostAudioUnavailable,
@@ -1538,6 +1744,10 @@ final class KeyboardFlowCoordinator {
state.phase = currentUtteranceRequest?.isEdit == true
? .requestingPermissions
: .recording
if currentUtteranceRequest?.isAIQuestion == true,
let currentUtteranceId {
onAIRecordingStarted(currentUtteranceId)
}
recomputeMicVoiceAvailability()
if let view = wakeLockView() {
ExtensionScreenWakeLock.acquire(from: view)
@@ -1658,11 +1868,19 @@ final class KeyboardFlowCoordinator {
guard isFlowRecording || isAwaitingFlowResult else { return }
switch state.phase {
case .recording, .processing:
if let result = matchingResult(),
result.status == .partial || result.status == .rawReady,
guard let result = matchingResult() else { return }
if result.status == .streaming,
result.resolvedUtteranceMode == .aiQuestion {
onAIStreamingAnswer(result.text ?? "", result.utteranceId)
return
}
if (result.status == .partial || result.status == .rawReady),
let partial = result.text,
!partial.isEmpty {
state.lastTranscript = partial
if result.resolvedUtteranceMode == .aiQuestion {
onAITranscript(partial, result.utteranceId, result.status)
}
}
default:
break
@@ -1679,11 +1897,21 @@ final class KeyboardFlowCoordinator {
let isCancelledDictation = currentUtteranceId.map {
cancelledDictationUtteranceIDs.contains($0)
} ?? false
let resultTimeout = isCancelledEdit || isCancelledDictation
? FlowSessionKeys.utteranceStartBudget
: (currentUtteranceRequest?.isEdit != true
? FlowWatchdog.resultTimeout(engineMode: state.engineMode)
: FlowSessionKeys.editLastInputProcessingBudget)
let isCancelledAI = currentUtteranceId.map {
cancelledAIUtteranceIDs.contains($0)
} ?? false
let resultTimeout: TimeInterval
if isCancelledEdit || isCancelledDictation || isCancelledAI {
resultTimeout = FlowSessionKeys.utteranceStartBudget
} else if currentUtteranceRequest?.isAIQuestion == true {
resultTimeout = FlowSessionKeys.keyboardAIResultTimeout(
engineMode: state.engineMode
)
} else if currentUtteranceRequest?.isEdit == true {
resultTimeout = FlowSessionKeys.editLastInputProcessingBudget
} else {
resultTimeout = FlowWatchdog.resultTimeout(engineMode: state.engineMode)
}
debug("resultWatchdog started timeout=\(Int(resultTimeout))s engine=\(state.engineMode)")
flowWatchdogTask = Task { @MainActor [weak self] in
while let self, !Task.isCancelled {
@@ -1700,6 +1928,10 @@ final class KeyboardFlowCoordinator {
self.consumeCancelledEditResultIfNeeded(result) {
return
}
if let result = self.matchingResult(),
self.consumeCancelledAIResultIfNeeded(result) {
return
}
if let result = self.matchingResult(), result.status == .final, let text = result.text, !text.isEmpty {
self.isAwaitingFlowResult = false
self.stopFlowWatchdog()
@@ -1707,6 +1939,10 @@ final class KeyboardFlowCoordinator {
self.onEditResult(result)
return
}
if result.resolvedUtteranceMode == .aiQuestion {
self.completeAIResult(result)
return
}
self.textInserter.handleFlowTranscript(
TranscriptionDelivery(
text: text,
@@ -1749,6 +1985,10 @@ final class KeyboardFlowCoordinator {
self.state.phase = .idle
return
}
if result.resolvedUtteranceMode == .aiQuestion {
self.completeAIFailure(result)
return
}
FlowSessionBridge.writeAck(
FlowAck(
sessionId: result.sessionId,
@@ -1823,6 +2063,14 @@ final class KeyboardFlowCoordinator {
self.recomputeMicVoiceAvailability()
return
}
if let id = self.currentUtteranceId,
self.cancelledAIUtteranceIDs.remove(id) != nil {
self.resetEditTransportState()
self.state.phase = .idle
self.state.lastTranscript = ""
self.recomputeMicVoiceAvailability()
return
}
if self.currentUtteranceRequest?.isEdit == true {
self.isAwaitingFlowResult = false
self.stopFlowWatchdog()
@@ -1837,6 +2085,21 @@ final class KeyboardFlowCoordinator {
self.state.phase = .idle
return
}
if self.currentUtteranceRequest?.isAIQuestion == true {
let utteranceID = self.currentUtteranceId
self.isAwaitingFlowResult = false
self.stopFlowWatchdog()
self.writeCommand(.abort)
self.onAIFailure(
ExtL10n.string("keyboard.ai.error.requestTimeout"),
utteranceID
)
self.resetEditTransportState()
self.state.phase = .idle
self.state.lastTranscript = ""
self.recomputeMicVoiceAvailability()
return
}
if self.deliverRawFallbackIfAvailable(reason: "resultTimeout") {
return
}
@@ -98,6 +98,42 @@ final class KeyboardTextInserter {
OSGLog.keyboardExt.info("flow insert length=\(trimmed.count, privacy: .public)")
}
/// Insert one explicitly confirmed AI answer and enqueue history/statistics
/// only after the field mutation has been issued.
@discardableResult
func insertAIAnswer(_ answer: AIAnswer) -> Bool {
let trimmed = answer.text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return false }
let separator = DictationTextComposer.insertionSeparator(
previousContext: contextBeforeInput(),
insertion: trimmed
)
let inserted = separator + trimmed
insertText(inserted)
let mutation = HistoryMutation(
action: .append,
entryID: answer.id,
text: trimmed,
engineMode: state.engineMode,
source: .ai,
usageCategory: .ai
)
HistoryMutationOutbox.enqueue(mutation)
recordLastInsertion(
inserted,
displayText: trimmed,
historyEntryID: answer.id,
historyEntryRevision: 0,
pendingHistoryMutationID: mutation.id
)
state.lastTranscript = ""
state.level = 0
OSGLog.keyboardExt.info("AI answer insert length=\(trimmed.count, privacy: .public)")
return true
}
/// Roll back the last voice insertion when it is still at the caret.
func undoLastInsertion() {
if undoLastEditIfPossible() {
@@ -41,11 +41,17 @@ struct KeyboardSurfaceRoot: View {
onInsert: onInsert,
onDeleteBackward: onDeleteBackward
)
case .ai:
AIKeyboardView(
state: state,
typing: typing,
onInsert: onInsert
)
}
}
.animation(.easeInOut(duration: 0.15), value: state.surface)
.onChange(of: state.surface) { _, newSurface in
if newSurface == .voice {
if newSurface != .typing {
typing.leaveTypingMode()
}
}
+328
View File
@@ -0,0 +1,328 @@
// AIKeyboardView.swift
// OSGKeyboard · Keyboard Extension
//
// Temporary voice-to-AI surface. The latest answer remains visible while a
// follow-up is running and is inserted only through the explicit Send action.
import SwiftUI
import OSGKeyboardShared
struct AIKeyboardView: View {
private enum Layout {
static let contentHeight: CGFloat = 174
static let actionRowHeight: CGFloat = 55
static let actionButtonHeight: CGFloat = 50
static let actionButtonMaxWidth: CGFloat = 150
static let statusHeight: CGFloat = 20
}
@Environment(\.colorScheme) private var colorScheme
@ObservedObject var state: KeyboardState
@ObservedObject var typing: TypingSessionController
let onInsert: (String) -> Void
private var palette: ThemePalette {
colorScheme == .dark ? Palette.dark : Palette.light
}
var body: some View {
VStack(spacing: 0) {
topBar.frame(height: KeyboardTopBarMetrics.height)
answerArea.frame(height: resolvedAnswerHeight)
actionRow.frame(height: Layout.actionRowHeight)
}
.frame(maxWidth: KeyboardChromeLayout.voiceContentMaxWidth)
.padding(.vertical, 4)
.padding(.horizontal, KeyboardChromeLayout.horizontalInset)
.frame(maxWidth: .infinity)
.frame(height: resolvedHeight)
.environment(\.themePalette, palette)
}
private var resolvedHeight: CGFloat {
TypingSurfaceMetrics.contentHeight(
isIPad: state.usesIPadLayoutMetrics,
width: state.layoutWidth
)
}
private var resolvedAnswerHeight: CGFloat {
max(
Layout.contentHeight,
resolvedHeight
- KeyboardTopBarMetrics.height
- Layout.actionRowHeight
- 8
)
}
private var topBar: some View {
HStack(spacing: Spacing.xs) {
KeyboardBrandLogo(action: state.openSettings)
Spacer(minLength: 0)
if state.canCancelAIInput {
KeyboardCancelButton(
action: state.cancelAIInput,
accessibilityLabel: ExtL10n.text("keyboard.ai.cancel"),
accessibilityHint: ExtL10n.text("keyboard.ai.cancelHint")
)
} else {
KeyboardTopControls(
state: state,
typing: typing,
palette: palette,
onInsert: onInsert
)
}
}
.padding(.horizontal, KeyboardTopBarMetrics.nestedHorizontalInset)
}
private var answerArea: some View {
ZStack(alignment: .bottom) {
if showsPlaceholder {
// Empty-state tip: geometric center of the answer plane.
Text(ExtL10n.string("keyboard.ai.placeholder"))
.font(TypeStyle.body)
.foregroundStyle(palette.textTertiary)
.multilineTextAlignment(.center)
.padding(.horizontal, Spacing.md)
.frame(maxWidth: .infinity, maxHeight: .infinity)
} else {
ScrollViewReader { proxy in
ScrollView(.vertical) {
Group {
if let draft = state.aiSession.draftAnswerText,
!draft.isEmpty {
Text(draft)
.foregroundStyle(palette.textPrimary)
.id("ai-draft")
} else if let answer = state.aiSession.answer {
Text(answer.text)
.foregroundStyle(palette.textPrimary)
.id(answer.id)
}
}
.font(TypeStyle.body)
.frame(maxWidth: .infinity, alignment: .topLeading)
.padding(.horizontal, Spacing.md)
.padding(.top, Spacing.sm)
.padding(.bottom, Layout.statusHeight + Spacing.sm)
}
.scrollIndicators(.visible)
.onChange(of: state.aiSession.answer?.id) { _, answerID in
guard let answerID else { return }
proxy.scrollTo(answerID, anchor: .top)
}
.onChange(of: state.aiSession.draftAnswerText) { _, draft in
guard let draft, !draft.isEmpty else { return }
proxy.scrollTo("ai-draft", anchor: .bottom)
}
}
}
statusLine
.frame(height: Layout.statusHeight)
.padding(.horizontal, Spacing.md)
}
}
/// No draft/answer yet show the centered mic guidance instead of a scroll body.
private var showsPlaceholder: Bool {
let hasDraft = !(state.aiSession.draftAnswerText?.isEmpty ?? true)
return !hasDraft && state.aiSession.answer == nil
}
private var statusLine: some View {
// Loading spinner lives on the mic button only avoid a second
// ProgressView beside the status / draft caption.
Text(statusText)
.font(TypeStyle.caption)
.foregroundStyle(
state.aiSession.phase == .failed
? palette.warning
: palette.textSecondary
)
.lineLimit(1)
.truncationMode(.head)
.frame(maxWidth: .infinity, alignment: .center)
}
private var actionRow: some View {
HStack(spacing: Spacing.sm) {
aiMicrophoneButton
sendButton
}
.frame(maxWidth: .infinity)
}
private var aiMicrophoneButton: some View {
Button(action: state.tapAIMic) {
ZStack {
Capsule().fill(palette.accent)
if state.aiSession.phase == .listening {
Capsule()
.stroke(Color.white.opacity(0.28), lineWidth: 1.5)
.scaleEffect(1 + min(max(state.level, 0), 1) * 0.08)
.animation(Motion.soft, value: state.level)
}
microphoneContent
}
.frame(
maxWidth: Layout.actionButtonMaxWidth,
minHeight: Layout.actionButtonHeight,
maxHeight: Layout.actionButtonHeight
)
.contentShape(Capsule())
}
.buttonStyle(.plain)
.disabled(microphoneDisabled)
.accessibilityLabel(ExtL10n.text(microphoneAccessibilityKey))
}
@ViewBuilder
private var microphoneContent: some View {
switch state.aiSession.phase {
case .listening:
WaveformView(
level: state.level,
barCount: 7,
color: .white,
active: true
)
.frame(width: 35, height: 22)
.clipped()
case .preparing, .recognizing, .generating:
ProgressView().tint(.white)
case .inactive, .idle, .ready, .awaitingSend, .inserted, .sent, .failed:
Image(systemName: "mic.fill")
.font(.system(size: 21, weight: .semibold))
.foregroundStyle(.white)
}
}
private var sendButton: some View {
Button(action: state.sendAIAnswer) {
HStack(spacing: Spacing.xs) {
Image(systemName: answerActionSystemName)
Text(answerActionTitle)
}
.font(.system(size: 16, weight: .semibold))
.foregroundStyle(answerActionForeground)
.frame(
maxWidth: Layout.actionButtonMaxWidth,
minHeight: Layout.actionButtonHeight,
maxHeight: Layout.actionButtonHeight
)
.background(
answerActionFill,
in: Capsule()
)
.overlay(Capsule().stroke(answerActionBorder, lineWidth: 0.5))
.contentShape(Capsule())
}
.buttonStyle(.plain)
.disabled(!state.aiSession.canPerformAnswerAction)
.accessibilityLabel(Text(answerActionTitle))
.accessibilityHint(ExtL10n.text("keyboard.ai.sendA11y"))
}
private var answerActionTitle: String {
switch state.aiSession.phase {
case .awaitingSend:
return ExtL10n.string("common.send")
case .inserted:
return ExtL10n.string("keyboard.ai.inserted")
case .sent:
return ExtL10n.string("keyboard.ai.sent")
case .inactive, .idle, .preparing, .listening, .recognizing,
.generating, .ready, .failed:
return ExtL10n.string("keyboard.ai.insert")
}
}
private var answerActionSystemName: String {
switch state.aiSession.phase {
case .awaitingSend:
return "paperplane.fill"
case .inserted, .sent:
return "checkmark"
case .inactive, .idle, .preparing, .listening, .recognizing,
.generating, .ready, .failed:
return "plus"
}
}
private var answerActionFill: Color {
guard state.aiSession.canPerformAnswerAction else {
return palette.surfaceElevated
}
return state.aiSession.canSend
? palette.accent
: NativeKeyboardKeyColors.fill(for: colorScheme)
}
private var answerActionForeground: Color {
guard state.aiSession.canPerformAnswerAction else {
return palette.textTertiary
}
return state.aiSession.canSend
? .white
: NativeKeyboardKeyColors.text(for: colorScheme)
}
private var answerActionBorder: Color {
guard state.aiSession.canSend else {
return palette.divider
}
return Color.black.opacity(colorScheme == .dark ? 0.10 : 0.08)
}
private var microphoneDisabled: Bool {
switch state.aiSession.phase {
case .preparing, .recognizing, .generating:
return true
case .inactive, .idle, .listening, .ready, .awaitingSend,
.inserted, .sent, .failed:
return state.micDisabled || !state.aiServiceAvailable
}
}
private var statusText: String {
if let error = state.aiSession.errorMessage, !error.isEmpty {
return error
}
if !state.aiServiceAvailable {
return ExtL10n.string("keyboard.ai.error.missingAPIKey")
}
switch state.aiSession.phase {
case .inactive, .idle, .ready, .awaitingSend, .inserted, .sent:
return ""
case .preparing:
return ExtL10n.string("keyboard.placeholder.preparing")
case .listening:
return state.aiSession.transcript.isEmpty
? ExtL10n.string("keyboard.ai.listening")
: state.aiSession.transcript
case .recognizing:
return state.aiSession.transcript.isEmpty
? ExtL10n.string("keyboard.ai.recognizing")
: state.aiSession.transcript
case .generating:
if let draft = state.aiSession.draftAnswerText, !draft.isEmpty {
return ExtL10n.string("keyboard.ai.generating")
}
return state.aiSession.transcript.isEmpty
? ExtL10n.string("keyboard.ai.thinking")
: state.aiSession.transcript
case .failed:
return ExtL10n.string("keyboard.ai.error.requestFailed")
}
}
private var microphoneAccessibilityKey: String {
state.aiSession.phase == .listening
? "keyboard.ai.stopA11y"
: "keyboard.ai.startA11y"
}
}
+14 -3
View File
@@ -613,9 +613,16 @@ private struct TranscriptLine: View {
Group {
switch micVoiceAvailability {
case .ready:
ExtL10n.text("keyboard.placeholder.idle")
// Soft tip when polish key is missing but local ASR can still run.
if !micDisabledHint.isEmpty {
Text(micDisabledHint)
} else {
ExtL10n.text("keyboard.placeholder.idle")
}
case .unavailable(.missingAPIKey):
Text(micDisabledHint)
Text(micDisabledHint.isEmpty
? ExtL10n.string("keyboard.mic.disabled.missingApiKey")
: micDisabledHint)
case .unavailable(.hostNotReady):
ExtL10n.text("keyboard.placeholder.idle")
case .unavailable(.preparingSession):
@@ -631,7 +638,11 @@ private struct TranscriptLine: View {
}
}
.font(TypeStyle.caption)
.foregroundStyle(isWarning ? palette.warning : palette.textTertiary)
.foregroundStyle(
(isWarning || (micVoiceAvailability.isReady && !micDisabledHint.isEmpty))
? palette.warning
: palette.textTertiary
)
.lineLimit(1)
.truncationMode(.tail)
}
+11 -1
View File
@@ -73,12 +73,14 @@ struct KeyboardCancelButton: View {
}
private enum KeyboardInputTab: CaseIterable {
case ai
case voice
case chinese
case english
var title: String {
switch self {
case .ai: return "AI"
case .voice: return "语音"
case .chinese: return "中文"
case .english: return "EN"
@@ -107,7 +109,10 @@ struct KeyboardTopControls: View {
.foregroundStyle(
isSelected(tab) ? palette.textPrimary : palette.textSecondary
)
.frame(width: tab == .english ? 34 : 42, height: 30)
.frame(
width: tab == .english || tab == .ai ? 34 : 42,
height: 30
)
.background {
if isSelected(tab) {
Capsule()
@@ -163,6 +168,8 @@ struct KeyboardTopControls: View {
private func isSelected(_ tab: KeyboardInputTab) -> Bool {
switch tab {
case .ai:
return state.surface == .ai
case .voice:
return state.surface == .voice
case .chinese:
@@ -174,6 +181,8 @@ struct KeyboardTopControls: View {
private func select(_ tab: KeyboardInputTab) {
switch tab {
case .ai:
state.setSurface(.ai)
case .voice:
state.setSurface(.voice)
case .chinese:
@@ -202,6 +211,7 @@ struct KeyboardTopControls: View {
private func accessibilityLabel(for tab: KeyboardInputTab) -> String {
switch tab {
case .ai: return "切换到 AI 问答"
case .voice: return "切换到语音输入"
case .chinese: return "切换到中文输入"
case .english: return "切换到英文输入"
+27 -4
View File
@@ -11,9 +11,9 @@
"onboarding.enable.step4" = "Allow Full Access is required for the microphone and LLM calls.";
"onboarding.enable.openSettings" = "Open iOS Settings";
"onboarding.api.title" = "Choose engine";
"onboarding.api.subtitle" = "Local needs no API key; Cloud polishes your text via LLM.";
"onboarding.api.localReady.title" = "No API key needed";
"onboarding.api.localReady.body" = "Local recognition is ready to use. Tap Done to start.";
"onboarding.api.subtitle" = "Local ASR needs no key; add an API key to enable AI polish.";
"onboarding.api.localReady.title" = "Local ASR is ready";
"onboarding.api.localReady.body" = "Recognition works on-device. Add an API key in Settings for AI polish.";
/* Common navigation */
"common.back" = "Back";
@@ -168,7 +168,8 @@
"keyboard.error.manualOpenDictate" = "System blocked the jump. Open OSGKeyboard to record, then return.";
"keyboard.error.llm.noApiKey" = "API key missing · configure it in the main app";
"keyboard.mic.disabled.missingApiKey" = "Fill in API key in Settings first";
"keyboard.error.llm.localPolishUnavailable" = "Built-in polish unavailable · inserted raw text";
"keyboard.mic.hint.missingPolishApiKey" = "Add an API key in Settings to enable polish";
"keyboard.error.llm.localPolishUnavailable" = "API key missing · inserted raw text";
"keyboard.error.llm.unauthorized" = "Invalid API key (401) · check main app settings";
"keyboard.error.llm.rateLimited" = "Rate limited (429) · try again later";
@@ -261,3 +262,25 @@
"keyboard.edit.apply" = "Apply edit";
"keyboard.edit.append" = "Insert at cursor";
"keyboard.edit.stop" = "Finish editing instruction";
/* AI question mode */
"keyboard.ai.placeholder" = "Tap the microphone to ask AI";
"keyboard.ai.hint" = "Insert the AI answer, then tap Send";
"keyboard.ai.listening" = "Listening…";
"keyboard.ai.recognizing" = "Recognizing your question…";
"keyboard.ai.generating" = "AI is answering…";
"keyboard.ai.thinking" = "AI is thinking…";
"keyboard.ai.send" = "Send";
"keyboard.ai.insert" = "Insert";
"keyboard.ai.inserted" = "Inserted";
"keyboard.ai.sent" = "Sent";
"keyboard.ai.cancel" = "Cancel AI question";
"keyboard.ai.cancelHint" = "Cancel the current recording, recognition, or AI request.";
"keyboard.ai.sendA11y" = "Tap once to insert the AI answer, then tap again to send in supported fields";
"keyboard.ai.startA11y" = "Start asking AI";
"keyboard.ai.stopA11y" = "Finish the question and send it to AI";
"keyboard.ai.error.missingAPIKey" = "Configure an AI service in the main app first";
"keyboard.ai.error.pipelineBusy" = "Voice input is busy. Try again shortly";
"keyboard.ai.error.startTimeout" = "Microphone startup timed out. Try again";
"keyboard.ai.error.requestTimeout" = "AI response timed out. Try again";
"keyboard.ai.error.requestFailed" = "AI response failed. Try again";
+27 -4
View File
@@ -11,9 +11,9 @@
"onboarding.enable.step4" = "允许完全访问是麦克风和网络调用的前提。";
"onboarding.enable.openSettings" = "打开 iOS 设置";
"onboarding.api.title" = "选择引擎";
"onboarding.api.subtitle" = "Local 不需要 API Key;Cloud 用 LLM 润色文字。";
"onboarding.api.localReady.title" = "无需配置 API Key";
"onboarding.api.localReady.body" = "本地识别直接可用,按 Done 即可开始使用。";
"onboarding.api.subtitle" = "本地识别无需 Key;填写 API Key 后可开启 AI 润色。";
"onboarding.api.localReady.title" = "本地识别已就绪";
"onboarding.api.localReady.body" = "识别在端侧完成。请在设置中填写 API Key 以开启 AI 润色。";
/* Common navigation */
"common.back" = "返回";
@@ -168,7 +168,8 @@
"keyboard.error.manualOpenDictate" = "系统拒绝了跳转,请手动打开 OSGKeyboard 录音";
"keyboard.error.llm.noApiKey" = "未配置 API Key · 请在主 App 设置中填写";
"keyboard.mic.disabled.missingApiKey" = "请先在设置中填写 API Key";
"keyboard.error.llm.localPolishUnavailable" = "内置润色不可用 · 已插入原始文本";
"keyboard.mic.hint.missingPolishApiKey" = "请先在设置中填写 API Key,才能润色";
"keyboard.error.llm.localPolishUnavailable" = "未填写 API Key · 已插入原始文本";
"keyboard.error.llm.unauthorized" = "API Key 无效 (401) · 请检查主 App 设置";
"keyboard.error.llm.rateLimited" = "API 限流 (429) · 请稍后再试";
@@ -261,3 +262,25 @@
"keyboard.edit.apply" = "应用编辑";
"keyboard.edit.append" = "插入当前位置";
"keyboard.edit.stop" = "完成编辑指令";
/* AI 问答模式 */
"keyboard.ai.placeholder" = "点击麦克风向 AI 提问";
"keyboard.ai.hint" = "先插入 AI 回答,再按发送";
"keyboard.ai.listening" = "正在聆听…";
"keyboard.ai.recognizing" = "正在识别问题…";
"keyboard.ai.generating" = "AI 正在回答…";
"keyboard.ai.thinking" = "AI 正在思考…";
"keyboard.ai.send" = "发送";
"keyboard.ai.insert" = "插入";
"keyboard.ai.inserted" = "已插入";
"keyboard.ai.sent" = "已发送";
"keyboard.ai.cancel" = "取消 AI 问答";
"keyboard.ai.cancelHint" = "取消当前录音、识别或 AI 请求。";
"keyboard.ai.sendA11y" = "首次点击插入 AI 回答;在支持发送的输入框中再次点击发送";
"keyboard.ai.startA11y" = "开始向 AI 提问";
"keyboard.ai.stopA11y" = "结束提问并发送给 AI";
"keyboard.ai.error.missingAPIKey" = "请先在主 App 配置可用的 AI 服务";
"keyboard.ai.error.pipelineBusy" = "语音服务正忙,请稍后重试";
"keyboard.ai.error.startTimeout" = "麦克风启动超时,请重试";
"keyboard.ai.error.requestTimeout" = "AI 回答超时,请重试";
"keyboard.ai.error.requestFailed" = "AI 回答失败,请重试";