feat(keyboard): ship AI mode surface with streaming search answers
Add the AI keyboard tab, Agent settings, and user-owned LLM key path for 1.7.0, including streaming answers and web-search transports without the built-in DeepSeek fallback.
This commit is contained in:
@@ -65,6 +65,7 @@ public final class KeyboardViewController: UIInputViewController {
|
||||
private var textInserter: KeyboardTextInserter!
|
||||
private var flowCoordinator: KeyboardFlowCoordinator!
|
||||
private var lastInputEditCoordinator: LastInputEditCoordinator!
|
||||
private var aiKeyboardCoordinator: AIKeyboardCoordinator!
|
||||
private var configSync: KeyboardConfigSync!
|
||||
/// UIKit may synchronously lay out the view during `viewDidLoad`.
|
||||
/// Keep this optional so an early layout pass is harmless.
|
||||
@@ -178,6 +179,12 @@ public final class KeyboardViewController: UIInputViewController {
|
||||
TypingInputConfiguration.persistLastSurface(
|
||||
preserve ? .voice : state.surface
|
||||
)
|
||||
if state.surface == .ai {
|
||||
// AI context never survives a keyboard presentation, but the
|
||||
// selected surface itself is restored on the next open.
|
||||
TypingInputConfiguration.persistLastSurface(.ai)
|
||||
aiKeyboardCoordinator.leave()
|
||||
}
|
||||
if !preserve {
|
||||
prepareSurfaceForNextPresentation()
|
||||
}
|
||||
@@ -375,6 +382,44 @@ public final class KeyboardViewController: UIInputViewController {
|
||||
flowCoordinator.onEditFailure = { [weak self] message in
|
||||
self?.lastInputEditCoordinator.fail(message)
|
||||
}
|
||||
aiKeyboardCoordinator = AIKeyboardCoordinator(
|
||||
state: state,
|
||||
flow: flowCoordinator,
|
||||
insertAnswer: { [weak self] answer in
|
||||
self?.textInserter.insertAIAnswer(answer) ?? false
|
||||
},
|
||||
performReturn: { [weak self] in
|
||||
self?.textDocumentProxy.insertText("\n")
|
||||
}
|
||||
)
|
||||
flowCoordinator.onAIUtterancePrepared = { [weak self] utteranceID in
|
||||
self?.aiKeyboardCoordinator.utterancePrepared(utteranceID)
|
||||
}
|
||||
flowCoordinator.onAIRecordingStarted = { [weak self] utteranceID in
|
||||
self?.aiKeyboardCoordinator.recordingStarted(utteranceID)
|
||||
}
|
||||
flowCoordinator.onAIRecognitionStarted = { [weak self] utteranceID in
|
||||
self?.aiKeyboardCoordinator.recognitionStarted(utteranceID)
|
||||
}
|
||||
flowCoordinator.onAITranscript = { [weak self] transcript, utteranceID, status in
|
||||
self?.aiKeyboardCoordinator.receiveTranscript(
|
||||
transcript,
|
||||
utteranceID: utteranceID,
|
||||
status: status
|
||||
)
|
||||
}
|
||||
flowCoordinator.onAIStreamingAnswer = { [weak self] draft, utteranceID in
|
||||
self?.aiKeyboardCoordinator.receivePartialAnswer(
|
||||
draft,
|
||||
utteranceID: utteranceID
|
||||
)
|
||||
}
|
||||
flowCoordinator.onAIResult = { [weak self] result in
|
||||
self?.aiKeyboardCoordinator.receive(result: result)
|
||||
}
|
||||
flowCoordinator.onAIFailure = { [weak self] message, utteranceID in
|
||||
self?.aiKeyboardCoordinator.fail(message, utteranceID: utteranceID)
|
||||
}
|
||||
_ = textInserter.recoverPendingEditTransactionIfNeeded()
|
||||
|
||||
cursorDrag = CursorDragController(
|
||||
@@ -409,6 +454,15 @@ public final class KeyboardViewController: UIInputViewController {
|
||||
state.closeEditMode = { [weak self] in
|
||||
self?.lastInputEditCoordinator.close()
|
||||
}
|
||||
state.tapAIMic = { [weak self] in
|
||||
self?.aiKeyboardCoordinator.toggleMicrophone()
|
||||
}
|
||||
state.cancelAIInput = { [weak self] in
|
||||
self?.aiKeyboardCoordinator.cancel()
|
||||
}
|
||||
state.sendAIAnswer = { [weak self] in
|
||||
self?.aiKeyboardCoordinator.sendLatestAnswer()
|
||||
}
|
||||
state.openSettings = { [weak self] in self?.openHostApp() }
|
||||
state.openInputMethodSetup = { [weak self] in self?.openHostApp(path: "deployrime") }
|
||||
// The globe UIButton registers this controller's standard
|
||||
@@ -471,6 +525,9 @@ public final class KeyboardViewController: UIInputViewController {
|
||||
return
|
||||
}
|
||||
guard state.surface != surface else {
|
||||
if surface == .ai {
|
||||
aiKeyboardCoordinator.enterIfNeeded()
|
||||
}
|
||||
refreshKeyboardHeight()
|
||||
return
|
||||
}
|
||||
@@ -478,11 +535,18 @@ public final class KeyboardViewController: UIInputViewController {
|
||||
"applySurface \(state.surface.rawValue) → \(surface.rawValue) \(OSGDiag.memoryTag())",
|
||||
category: "boot"
|
||||
)
|
||||
let previousSurface = state.surface
|
||||
if previousSurface == .ai, surface != .ai {
|
||||
aiKeyboardCoordinator.leave()
|
||||
}
|
||||
state.surface = surface
|
||||
if surface == .voice {
|
||||
typingSession.leaveTypingMode()
|
||||
} else {
|
||||
if surface == .typing {
|
||||
typingSession.enterTypingMode()
|
||||
} else {
|
||||
typingSession.leaveTypingMode()
|
||||
}
|
||||
if surface == .ai {
|
||||
aiKeyboardCoordinator.enterIfNeeded()
|
||||
}
|
||||
refreshKeyboardHeight()
|
||||
}
|
||||
@@ -500,6 +564,9 @@ public final class KeyboardViewController: UIInputViewController {
|
||||
category: "boot"
|
||||
)
|
||||
applySurface(resolved)
|
||||
if resolved == .ai {
|
||||
aiKeyboardCoordinator.beginNewPresentation()
|
||||
}
|
||||
}
|
||||
|
||||
/// When not remembering, snap to the static open preference while hidden
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
// AIKeyboardCoordinator.swift
|
||||
// OSGKeyboard · Keyboard Extension
|
||||
//
|
||||
// Owns the temporary AI-mode UI state. Audio, ASR, and LLM work remain in the
|
||||
// shared Flow transport and host app; this coordinator never performs network
|
||||
// work and never inserts an answer before explicit user confirmation.
|
||||
|
||||
import Foundation
|
||||
import OSGKeyboardShared
|
||||
|
||||
@MainActor
|
||||
final class AIKeyboardCoordinator {
|
||||
private let state: KeyboardState
|
||||
private let flow: KeyboardFlowCoordinator
|
||||
private let insertAnswer: (AIAnswer) -> Bool
|
||||
private let performReturn: () -> Void
|
||||
|
||||
init(
|
||||
state: KeyboardState,
|
||||
flow: KeyboardFlowCoordinator,
|
||||
insertAnswer: @escaping (AIAnswer) -> Bool,
|
||||
performReturn: @escaping () -> Void
|
||||
) {
|
||||
self.state = state
|
||||
self.flow = flow
|
||||
self.insertAnswer = insertAnswer
|
||||
self.performReturn = performReturn
|
||||
}
|
||||
|
||||
func beginNewPresentation() {
|
||||
endConversationIfNeeded()
|
||||
state.aiSession.enter()
|
||||
}
|
||||
|
||||
func enterIfNeeded() {
|
||||
guard !state.aiSession.isActive else { return }
|
||||
state.aiSession.enter()
|
||||
}
|
||||
|
||||
func leave() {
|
||||
if state.aiSession.isBusy {
|
||||
flow.cancelAIRecording()
|
||||
}
|
||||
endConversationIfNeeded()
|
||||
state.aiSession.leave()
|
||||
}
|
||||
|
||||
func toggleMicrophone() {
|
||||
switch state.aiSession.phase {
|
||||
case .listening:
|
||||
guard let utteranceID = state.aiSession.activeUtteranceID else { return }
|
||||
state.aiSession.beginRecognizing(utteranceID: utteranceID)
|
||||
flow.stopAIRecording()
|
||||
case .idle, .ready, .awaitingSend, .inserted, .sent, .failed:
|
||||
enterIfNeeded()
|
||||
guard let conversationID = state.aiSession.conversationID else { return }
|
||||
let disposition = flow.beginAIRecording(conversationID: conversationID)
|
||||
if case .rejected(let rejection) = disposition {
|
||||
state.aiSession.fail(message(for: rejection), utteranceID: nil)
|
||||
}
|
||||
case .inactive:
|
||||
enterIfNeeded()
|
||||
toggleMicrophone()
|
||||
case .preparing, .recognizing, .generating:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
guard state.aiSession.isBusy else { return }
|
||||
flow.cancelAIRecording()
|
||||
state.aiSession.cancelCurrentWork()
|
||||
}
|
||||
|
||||
func sendLatestAnswer() {
|
||||
if state.aiSession.canInsert, let answer = state.aiSession.answer {
|
||||
guard insertAnswer(answer) else { return }
|
||||
state.aiSession.markAnswerInserted(
|
||||
offersSend: state.returnKeyRole == .send
|
||||
)
|
||||
} else if state.aiSession.canSend {
|
||||
state.aiSession.markAnswerSent()
|
||||
// Let the host consume the inserted answer before issuing Return.
|
||||
Task { @MainActor [weak self] in
|
||||
await Task.yield()
|
||||
self?.performReturn()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func utterancePrepared(_ utteranceID: UUID) {
|
||||
state.aiSession.beginPreparing(utteranceID: utteranceID)
|
||||
}
|
||||
|
||||
func recordingStarted(_ utteranceID: UUID) {
|
||||
state.aiSession.beginListening(utteranceID: utteranceID)
|
||||
}
|
||||
|
||||
func recognitionStarted(_ utteranceID: UUID) {
|
||||
state.aiSession.beginRecognizing(utteranceID: utteranceID)
|
||||
}
|
||||
|
||||
func receiveTranscript(
|
||||
_ transcript: String,
|
||||
utteranceID: UUID,
|
||||
status: FlowResult.Status
|
||||
) {
|
||||
state.aiSession.updateTranscript(transcript, utteranceID: utteranceID)
|
||||
if status == .rawReady {
|
||||
state.aiSession.beginGenerating(
|
||||
question: transcript,
|
||||
utteranceID: utteranceID
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func receivePartialAnswer(_ draft: String, utteranceID: UUID) {
|
||||
state.aiSession.receivePartialAnswer(draft, utteranceID: utteranceID)
|
||||
}
|
||||
|
||||
func receive(result: FlowResult) {
|
||||
guard result.resolvedUtteranceMode == .aiQuestion else {
|
||||
return
|
||||
}
|
||||
guard result.aiConversationID == state.aiSession.conversationID,
|
||||
let answer = result.text,
|
||||
!answer.isEmpty else {
|
||||
state.aiSession.fail(
|
||||
ExtL10n.string("keyboard.ai.error.requestFailed"),
|
||||
utteranceID: result.utteranceId
|
||||
)
|
||||
return
|
||||
}
|
||||
state.aiSession.receiveAnswer(answer, utteranceID: result.utteranceId)
|
||||
}
|
||||
|
||||
func fail(_ message: String, utteranceID: UUID?) {
|
||||
state.aiSession.fail(message, utteranceID: utteranceID)
|
||||
}
|
||||
|
||||
private func endConversationIfNeeded() {
|
||||
guard let conversationID = state.aiSession.conversationID else { return }
|
||||
flow.endAIConversation(conversationID)
|
||||
}
|
||||
|
||||
private func message(for rejection: FlowUtteranceStartRejection) -> String {
|
||||
switch rejection {
|
||||
case .onboardingIncomplete:
|
||||
return ExtL10n.string("keyboard.hint.finishSetupInApp")
|
||||
case .missingAPIKey:
|
||||
return ExtL10n.string("keyboard.ai.error.missingAPIKey")
|
||||
case .noFullAccess:
|
||||
return ExtL10n.string("keyboard.error.fullAccessRequired")
|
||||
case .appGroupUnavailable:
|
||||
return ExtL10n.string("keyboard.error.appGroupCommunication")
|
||||
case .hostUnavailable:
|
||||
return ExtL10n.string("keyboard.flow.hostDisconnected")
|
||||
case .pipelineBusy:
|
||||
return ExtL10n.string("keyboard.ai.error.pipelineBusy")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -42,10 +42,7 @@ public struct AppGroupPersistor {
|
||||
state.handednessPreference = store.handednessPreference
|
||||
state.cursorDragNavigationEnabled = store.cursorDragNavigationEnabled
|
||||
state.keyboardHapticIntensity = store.keyboardHapticIntensity
|
||||
state.micDisabled = store.isCloudAPIKeyMissingForVoiceInput
|
||||
state.micDisabledHint = store.isCloudAPIKeyMissingForVoiceInput
|
||||
? ExtL10n.string("keyboard.mic.disabled.missingApiKey")
|
||||
: ""
|
||||
applyAPIKeyAvailability(store: store, into: state)
|
||||
|
||||
#if DEBUG
|
||||
// Print a masked view of the live App Group config so we can see
|
||||
@@ -95,10 +92,26 @@ public struct AppGroupPersistor {
|
||||
state.handednessPreference = store.handednessPreference
|
||||
state.cursorDragNavigationEnabled = store.cursorDragNavigationEnabled
|
||||
state.keyboardHapticIntensity = store.keyboardHapticIntensity
|
||||
state.micDisabled = store.isCloudAPIKeyMissingForVoiceInput
|
||||
state.micDisabledHint = store.isCloudAPIKeyMissingForVoiceInput
|
||||
? ExtL10n.string("keyboard.mic.disabled.missingApiKey")
|
||||
: ""
|
||||
applyAPIKeyAvailability(store: store, into: state)
|
||||
}
|
||||
|
||||
/// Cloud without ASR/LLM keys blocks the mic. Local ASR still works when
|
||||
/// the polish key is missing — show a soft tip above the mic instead.
|
||||
private func applyAPIKeyAvailability(
|
||||
store: AppGroupStore,
|
||||
into state: KeyboardViewController.State
|
||||
) {
|
||||
state.aiServiceAvailable = !store.isPolishKeyMissing
|
||||
if store.isCloudAPIKeyMissingForVoiceInput {
|
||||
state.micDisabled = true
|
||||
state.micDisabledHint = ExtL10n.string("keyboard.mic.disabled.missingApiKey")
|
||||
} else if store.isPolishKeyMissing {
|
||||
state.micDisabled = false
|
||||
state.micDisabledHint = ExtL10n.string("keyboard.mic.hint.missingPolishApiKey")
|
||||
} else {
|
||||
state.micDisabled = false
|
||||
state.micDisabledHint = ""
|
||||
}
|
||||
}
|
||||
|
||||
/// Persist `mode` to the App Group store.
|
||||
|
||||
@@ -54,9 +54,17 @@ final class KeyboardFlowCoordinator {
|
||||
private var editHostConfirmed = false
|
||||
private var cancelledEditUtteranceIDs: Set<UUID> = []
|
||||
private var cancelledDictationUtteranceIDs: Set<UUID> = []
|
||||
private var cancelledAIUtteranceIDs: Set<UUID> = []
|
||||
var onEditHostRecordingConfirmed: () -> Void = {}
|
||||
var onEditResult: (FlowResult) -> Void = { _ in }
|
||||
var onEditFailure: (String) -> Void = { _ in }
|
||||
var onAIUtterancePrepared: (UUID) -> Void = { _ in }
|
||||
var onAIRecordingStarted: (UUID) -> Void = { _ in }
|
||||
var onAIRecognitionStarted: (UUID) -> Void = { _ in }
|
||||
var onAITranscript: (String, UUID, FlowResult.Status) -> Void = { _, _, _ in }
|
||||
var onAIStreamingAnswer: (String, UUID) -> Void = { _, _ in }
|
||||
var onAIResult: (FlowResult) -> Void = { _ in }
|
||||
var onAIFailure: (String, UUID?) -> Void = { _, _ in }
|
||||
/// Utterance whose final result we already inserted (or failed). Prevents
|
||||
/// `adoptHostBusyStateIfNeeded` from re-entering `.processing` after a
|
||||
/// stale App Group snapshot still says `reason=processing`.
|
||||
@@ -178,6 +186,7 @@ final class KeyboardFlowCoordinator {
|
||||
adoptPendingResultIfNeeded()
|
||||
consumePendingFlowDeliveryIfNeeded()
|
||||
consumeEditStartFailureIfNeeded()
|
||||
consumeAIStartFailureIfNeeded()
|
||||
|
||||
recoverFromDeadHostIfNeeded()
|
||||
|
||||
@@ -243,6 +252,15 @@ final class KeyboardFlowCoordinator {
|
||||
completeEditResult(result, outcome: .rejected)
|
||||
}
|
||||
|
||||
private func consumeAIStartFailureIfNeeded() {
|
||||
guard currentUtteranceRequest?.isAIQuestion == true,
|
||||
let result = matchingResult(),
|
||||
isTerminalFailure(result) else {
|
||||
return
|
||||
}
|
||||
completeAIFailure(result)
|
||||
}
|
||||
|
||||
private func recomputeMicVoiceAvailability() {
|
||||
FlowSessionBridge.reloadFromDisk()
|
||||
let readySnapshot = FlowSessionBridge.readySnapshot()
|
||||
@@ -617,13 +635,76 @@ final class KeyboardFlowCoordinator {
|
||||
startUtterance(.editLastInput(reference))
|
||||
}
|
||||
|
||||
func beginAIRecording(
|
||||
conversationID: UUID
|
||||
) -> FlowUtteranceStartDisposition {
|
||||
startUtterance(.aiQuestion(conversationID: conversationID))
|
||||
}
|
||||
|
||||
func stopAIRecording() {
|
||||
guard currentUtteranceRequest?.isAIQuestion == true else { return }
|
||||
pressEnded()
|
||||
}
|
||||
|
||||
func cancelAIRecording() {
|
||||
guard currentUtteranceRequest?.isAIQuestion == true else { return }
|
||||
recordWhenHostReady = false
|
||||
recordAfterHandoff = false
|
||||
isPendingFlowStart = false
|
||||
flowStartDeadline = 0
|
||||
coldStartDebouncer.reset()
|
||||
stopHostReadyWait()
|
||||
stopFlowWatchdog()
|
||||
stopUtteranceCountdown()
|
||||
ExtensionScreenWakeLock.release()
|
||||
state.level = 0
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
|
||||
guard let utteranceID = currentUtteranceId,
|
||||
isFlowRecording || isAwaitingFlowResult else {
|
||||
clearUnissuedUtterance()
|
||||
isFlowRecording = false
|
||||
isAwaitingFlowResult = false
|
||||
recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
|
||||
cancelledAIUtteranceIDs.insert(utteranceID)
|
||||
writeCommand(.abort)
|
||||
isFlowRecording = false
|
||||
isAwaitingFlowResult = true
|
||||
startFlowResultWatchdog()
|
||||
recomputeMicVoiceAvailability()
|
||||
}
|
||||
|
||||
func endAIConversation(_ conversationID: UUID) {
|
||||
guard let sessionID = FlowSessionBridge.readySnapshot()?.sessionId
|
||||
?? activeSessionId else {
|
||||
return
|
||||
}
|
||||
FlowSessionBridge.writeCommand(
|
||||
FlowCommand(
|
||||
sessionId: sessionID,
|
||||
utteranceId: UUID(),
|
||||
commandSeq: nextCommandSeq(),
|
||||
action: .endAIConversation,
|
||||
localeId: state.localeId,
|
||||
aiConversationID: conversationID
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
func stopEditRecording() {
|
||||
guard currentUtteranceRequest?.isEdit == true else { return }
|
||||
pressEnded()
|
||||
}
|
||||
|
||||
func cancelCurrentDictation() {
|
||||
guard currentUtteranceRequest?.isEdit != true else { return }
|
||||
guard currentUtteranceRequest?.isEdit != true,
|
||||
currentUtteranceRequest?.isAIQuestion != true else {
|
||||
return
|
||||
}
|
||||
|
||||
recordWhenHostReady = false
|
||||
recordAfterHandoff = false
|
||||
@@ -733,6 +814,50 @@ final class KeyboardFlowCoordinator {
|
||||
resetEditTransportState()
|
||||
}
|
||||
|
||||
private func completeAIResult(_ result: FlowResult) {
|
||||
onAIResult(result)
|
||||
FlowSessionBridge.writeAck(
|
||||
FlowAck(
|
||||
sessionId: result.sessionId,
|
||||
utteranceId: result.utteranceId,
|
||||
commandSeq: result.commandSeq,
|
||||
hostGeneration: result.hostGeneration,
|
||||
revision: result.revision
|
||||
)
|
||||
)
|
||||
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
||||
lastConsumedUtteranceId = result.utteranceId
|
||||
lastStoppedUtteranceId = nil
|
||||
resetEditTransportState()
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
recomputeMicVoiceAvailability()
|
||||
}
|
||||
|
||||
private func completeAIFailure(_ result: FlowResult) {
|
||||
onAIFailure(
|
||||
result.text ?? ExtL10n.string("keyboard.ai.error.requestFailed"),
|
||||
result.utteranceId
|
||||
)
|
||||
FlowSessionBridge.writeAck(
|
||||
FlowAck(
|
||||
sessionId: result.sessionId,
|
||||
utteranceId: result.utteranceId,
|
||||
commandSeq: result.commandSeq,
|
||||
hostGeneration: result.hostGeneration,
|
||||
revision: result.revision,
|
||||
deliveryOutcome: .rejected
|
||||
)
|
||||
)
|
||||
FlowSessionBridge.setPendingKeyboardUtteranceId(nil)
|
||||
lastConsumedUtteranceId = result.utteranceId
|
||||
lastStoppedUtteranceId = nil
|
||||
resetEditTransportState()
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
recomputeMicVoiceAvailability()
|
||||
}
|
||||
|
||||
func pressBegan() {
|
||||
_ = startUtterance(.dictation)
|
||||
}
|
||||
@@ -762,6 +887,9 @@ final class KeyboardFlowCoordinator {
|
||||
currentUtteranceId = utteranceID
|
||||
currentUtteranceRequest = request
|
||||
editHostConfirmed = false
|
||||
if request.isAIQuestion {
|
||||
onAIUtterancePrepared(utteranceID)
|
||||
}
|
||||
currentStartDeadlineAt = Date().timeIntervalSince1970
|
||||
+ FlowSessionKeys.utteranceStartBudget
|
||||
|
||||
@@ -863,7 +991,10 @@ final class KeyboardFlowCoordinator {
|
||||
writeCommand(.stopRecording)
|
||||
debug("pressEnded wrote stop command")
|
||||
state.phase = .processing
|
||||
if currentUtteranceRequest?.isEdit != true {
|
||||
if currentUtteranceRequest?.isAIQuestion == true,
|
||||
let currentUtteranceId {
|
||||
onAIRecognitionStarted(currentUtteranceId)
|
||||
} else if currentUtteranceRequest?.isEdit != true {
|
||||
state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
|
||||
}
|
||||
startFlowResultWatchdog()
|
||||
@@ -923,6 +1054,16 @@ final class KeyboardFlowCoordinator {
|
||||
state.lastTranscript = ""
|
||||
return
|
||||
}
|
||||
if currentUtteranceRequest?.isAIQuestion == true {
|
||||
onAIFailure(
|
||||
ExtL10n.string("keyboard.error.manualOpenForFlow"),
|
||||
currentUtteranceId
|
||||
)
|
||||
resetEditTransportState()
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
return
|
||||
}
|
||||
showManualOpenHint(path: "startflow")
|
||||
recomputeMicVoiceAvailability()
|
||||
return
|
||||
@@ -972,13 +1113,16 @@ final class KeyboardFlowCoordinator {
|
||||
commandSeq: nextCommandSeq(),
|
||||
action: action,
|
||||
localeId: state.localeId,
|
||||
fieldContext: action == .stopRecording ? fieldContextProvider() : nil,
|
||||
fieldContext: action == .stopRecording && !request.isAIQuestion
|
||||
? fieldContextProvider()
|
||||
: nil,
|
||||
utteranceMode: mode,
|
||||
editSourceText: action == .startRecording
|
||||
? request.editSourceText
|
||||
: nil,
|
||||
sourceHistoryEntryID: request.sourceHistoryEntryID,
|
||||
sourceHistoryEntryRevision: request.sourceHistoryEntryRevision,
|
||||
aiConversationID: request.aiConversationID,
|
||||
startDeadlineAt: action == .startRecording ? currentStartDeadlineAt : nil,
|
||||
processingDeadlineAt: action == .stopRecording && request.isEdit
|
||||
? Date().timeIntervalSince1970
|
||||
@@ -1027,6 +1171,10 @@ final class KeyboardFlowCoordinator {
|
||||
consumeCancelledEditResultIfNeeded(result) {
|
||||
return
|
||||
}
|
||||
if let result = matchingResult(),
|
||||
consumeCancelledAIResultIfNeeded(result) {
|
||||
return
|
||||
}
|
||||
if let result = matchingResult(), result.status == .final, let text = result.text, !text.isEmpty {
|
||||
isAwaitingFlowResult = false
|
||||
stopFlowWatchdog()
|
||||
@@ -1035,6 +1183,10 @@ final class KeyboardFlowCoordinator {
|
||||
onEditResult(result)
|
||||
return
|
||||
}
|
||||
if result.resolvedUtteranceMode == .aiQuestion {
|
||||
completeAIResult(result)
|
||||
return
|
||||
}
|
||||
textInserter.handleFlowTranscript(
|
||||
TranscriptionDelivery(
|
||||
text: text,
|
||||
@@ -1083,6 +1235,10 @@ final class KeyboardFlowCoordinator {
|
||||
state.phase = .idle
|
||||
return
|
||||
}
|
||||
if result.resolvedUtteranceMode == .aiQuestion {
|
||||
completeAIFailure(result)
|
||||
return
|
||||
}
|
||||
FlowSessionBridge.writeAck(
|
||||
FlowAck(
|
||||
sessionId: result.sessionId,
|
||||
@@ -1172,6 +1328,30 @@ final class KeyboardFlowCoordinator {
|
||||
return true
|
||||
}
|
||||
|
||||
private func consumeCancelledAIResultIfNeeded(_ result: FlowResult) -> Bool {
|
||||
guard cancelledAIUtteranceIDs.contains(result.utteranceId),
|
||||
result.status == .final || isTerminalFailure(result) else {
|
||||
return false
|
||||
}
|
||||
FlowSessionBridge.writeAck(
|
||||
FlowAck(
|
||||
sessionId: result.sessionId,
|
||||
utteranceId: result.utteranceId,
|
||||
commandSeq: result.commandSeq,
|
||||
hostGeneration: result.hostGeneration,
|
||||
revision: result.revision,
|
||||
deliveryOutcome: .rejected
|
||||
)
|
||||
)
|
||||
cancelledAIUtteranceIDs.remove(result.utteranceId)
|
||||
lastConsumedUtteranceId = result.utteranceId
|
||||
resetEditTransportState()
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
recomputeMicVoiceAvailability()
|
||||
return true
|
||||
}
|
||||
|
||||
private func adoptPendingResultIfNeeded() {
|
||||
guard !isAwaitingFlowResult, currentUtteranceId == nil,
|
||||
let pendingId = FlowSessionBridge.pendingKeyboardUtteranceId(),
|
||||
@@ -1343,6 +1523,18 @@ final class KeyboardFlowCoordinator {
|
||||
)
|
||||
return
|
||||
}
|
||||
if let id = currentUtteranceId,
|
||||
cancelledAIUtteranceIDs.remove(id) != nil {
|
||||
stopUtteranceCountdown()
|
||||
stopFlowWatchdog()
|
||||
ExtensionScreenWakeLock.release()
|
||||
resetEditTransportState()
|
||||
state.level = 0
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
if deliverRawFallbackIfAvailable(reason: "hostDisconnected") {
|
||||
return
|
||||
}
|
||||
@@ -1367,6 +1559,14 @@ final class KeyboardFlowCoordinator {
|
||||
recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
if currentUtteranceRequest?.isAIQuestion == true {
|
||||
onAIFailure(message, currentUtteranceId)
|
||||
resetEditTransportState()
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
state.phase = .error(.flowSessionExpired, message: message)
|
||||
scheduleAutoClearError()
|
||||
recomputeMicVoiceAvailability()
|
||||
@@ -1469,6 +1669,12 @@ final class KeyboardFlowCoordinator {
|
||||
if currentUtteranceRequest?.isEdit == true {
|
||||
onEditFailure(ExtL10n.string("keyboard.edit.error.startTimeout"))
|
||||
abortEditRecording()
|
||||
} else if currentUtteranceRequest?.isAIQuestion == true {
|
||||
onAIFailure(
|
||||
ExtL10n.string("keyboard.ai.error.startTimeout"),
|
||||
currentUtteranceId
|
||||
)
|
||||
cancelAIRecording()
|
||||
} else {
|
||||
state.phase = .error(
|
||||
.hostAudioUnavailable,
|
||||
@@ -1538,6 +1744,10 @@ final class KeyboardFlowCoordinator {
|
||||
state.phase = currentUtteranceRequest?.isEdit == true
|
||||
? .requestingPermissions
|
||||
: .recording
|
||||
if currentUtteranceRequest?.isAIQuestion == true,
|
||||
let currentUtteranceId {
|
||||
onAIRecordingStarted(currentUtteranceId)
|
||||
}
|
||||
recomputeMicVoiceAvailability()
|
||||
if let view = wakeLockView() {
|
||||
ExtensionScreenWakeLock.acquire(from: view)
|
||||
@@ -1658,11 +1868,19 @@ final class KeyboardFlowCoordinator {
|
||||
guard isFlowRecording || isAwaitingFlowResult else { return }
|
||||
switch state.phase {
|
||||
case .recording, .processing:
|
||||
if let result = matchingResult(),
|
||||
result.status == .partial || result.status == .rawReady,
|
||||
guard let result = matchingResult() else { return }
|
||||
if result.status == .streaming,
|
||||
result.resolvedUtteranceMode == .aiQuestion {
|
||||
onAIStreamingAnswer(result.text ?? "", result.utteranceId)
|
||||
return
|
||||
}
|
||||
if (result.status == .partial || result.status == .rawReady),
|
||||
let partial = result.text,
|
||||
!partial.isEmpty {
|
||||
state.lastTranscript = partial
|
||||
if result.resolvedUtteranceMode == .aiQuestion {
|
||||
onAITranscript(partial, result.utteranceId, result.status)
|
||||
}
|
||||
}
|
||||
default:
|
||||
break
|
||||
@@ -1679,11 +1897,21 @@ final class KeyboardFlowCoordinator {
|
||||
let isCancelledDictation = currentUtteranceId.map {
|
||||
cancelledDictationUtteranceIDs.contains($0)
|
||||
} ?? false
|
||||
let resultTimeout = isCancelledEdit || isCancelledDictation
|
||||
? FlowSessionKeys.utteranceStartBudget
|
||||
: (currentUtteranceRequest?.isEdit != true
|
||||
? FlowWatchdog.resultTimeout(engineMode: state.engineMode)
|
||||
: FlowSessionKeys.editLastInputProcessingBudget)
|
||||
let isCancelledAI = currentUtteranceId.map {
|
||||
cancelledAIUtteranceIDs.contains($0)
|
||||
} ?? false
|
||||
let resultTimeout: TimeInterval
|
||||
if isCancelledEdit || isCancelledDictation || isCancelledAI {
|
||||
resultTimeout = FlowSessionKeys.utteranceStartBudget
|
||||
} else if currentUtteranceRequest?.isAIQuestion == true {
|
||||
resultTimeout = FlowSessionKeys.keyboardAIResultTimeout(
|
||||
engineMode: state.engineMode
|
||||
)
|
||||
} else if currentUtteranceRequest?.isEdit == true {
|
||||
resultTimeout = FlowSessionKeys.editLastInputProcessingBudget
|
||||
} else {
|
||||
resultTimeout = FlowWatchdog.resultTimeout(engineMode: state.engineMode)
|
||||
}
|
||||
debug("resultWatchdog started timeout=\(Int(resultTimeout))s engine=\(state.engineMode)")
|
||||
flowWatchdogTask = Task { @MainActor [weak self] in
|
||||
while let self, !Task.isCancelled {
|
||||
@@ -1700,6 +1928,10 @@ final class KeyboardFlowCoordinator {
|
||||
self.consumeCancelledEditResultIfNeeded(result) {
|
||||
return
|
||||
}
|
||||
if let result = self.matchingResult(),
|
||||
self.consumeCancelledAIResultIfNeeded(result) {
|
||||
return
|
||||
}
|
||||
if let result = self.matchingResult(), result.status == .final, let text = result.text, !text.isEmpty {
|
||||
self.isAwaitingFlowResult = false
|
||||
self.stopFlowWatchdog()
|
||||
@@ -1707,6 +1939,10 @@ final class KeyboardFlowCoordinator {
|
||||
self.onEditResult(result)
|
||||
return
|
||||
}
|
||||
if result.resolvedUtteranceMode == .aiQuestion {
|
||||
self.completeAIResult(result)
|
||||
return
|
||||
}
|
||||
self.textInserter.handleFlowTranscript(
|
||||
TranscriptionDelivery(
|
||||
text: text,
|
||||
@@ -1749,6 +1985,10 @@ final class KeyboardFlowCoordinator {
|
||||
self.state.phase = .idle
|
||||
return
|
||||
}
|
||||
if result.resolvedUtteranceMode == .aiQuestion {
|
||||
self.completeAIFailure(result)
|
||||
return
|
||||
}
|
||||
FlowSessionBridge.writeAck(
|
||||
FlowAck(
|
||||
sessionId: result.sessionId,
|
||||
@@ -1823,6 +2063,14 @@ final class KeyboardFlowCoordinator {
|
||||
self.recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
if let id = self.currentUtteranceId,
|
||||
self.cancelledAIUtteranceIDs.remove(id) != nil {
|
||||
self.resetEditTransportState()
|
||||
self.state.phase = .idle
|
||||
self.state.lastTranscript = ""
|
||||
self.recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
if self.currentUtteranceRequest?.isEdit == true {
|
||||
self.isAwaitingFlowResult = false
|
||||
self.stopFlowWatchdog()
|
||||
@@ -1837,6 +2085,21 @@ final class KeyboardFlowCoordinator {
|
||||
self.state.phase = .idle
|
||||
return
|
||||
}
|
||||
if self.currentUtteranceRequest?.isAIQuestion == true {
|
||||
let utteranceID = self.currentUtteranceId
|
||||
self.isAwaitingFlowResult = false
|
||||
self.stopFlowWatchdog()
|
||||
self.writeCommand(.abort)
|
||||
self.onAIFailure(
|
||||
ExtL10n.string("keyboard.ai.error.requestTimeout"),
|
||||
utteranceID
|
||||
)
|
||||
self.resetEditTransportState()
|
||||
self.state.phase = .idle
|
||||
self.state.lastTranscript = ""
|
||||
self.recomputeMicVoiceAvailability()
|
||||
return
|
||||
}
|
||||
if self.deliverRawFallbackIfAvailable(reason: "resultTimeout") {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -98,6 +98,42 @@ final class KeyboardTextInserter {
|
||||
OSGLog.keyboardExt.info("flow insert length=\(trimmed.count, privacy: .public)")
|
||||
}
|
||||
|
||||
/// Insert one explicitly confirmed AI answer and enqueue history/statistics
|
||||
/// only after the field mutation has been issued.
|
||||
@discardableResult
|
||||
func insertAIAnswer(_ answer: AIAnswer) -> Bool {
|
||||
let trimmed = answer.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return false }
|
||||
|
||||
let separator = DictationTextComposer.insertionSeparator(
|
||||
previousContext: contextBeforeInput(),
|
||||
insertion: trimmed
|
||||
)
|
||||
let inserted = separator + trimmed
|
||||
insertText(inserted)
|
||||
|
||||
let mutation = HistoryMutation(
|
||||
action: .append,
|
||||
entryID: answer.id,
|
||||
text: trimmed,
|
||||
engineMode: state.engineMode,
|
||||
source: .ai,
|
||||
usageCategory: .ai
|
||||
)
|
||||
HistoryMutationOutbox.enqueue(mutation)
|
||||
recordLastInsertion(
|
||||
inserted,
|
||||
displayText: trimmed,
|
||||
historyEntryID: answer.id,
|
||||
historyEntryRevision: 0,
|
||||
pendingHistoryMutationID: mutation.id
|
||||
)
|
||||
state.lastTranscript = ""
|
||||
state.level = 0
|
||||
OSGLog.keyboardExt.info("AI answer insert length=\(trimmed.count, privacy: .public)")
|
||||
return true
|
||||
}
|
||||
|
||||
/// Roll back the last voice insertion when it is still at the caret.
|
||||
func undoLastInsertion() {
|
||||
if undoLastEditIfPossible() {
|
||||
|
||||
@@ -41,11 +41,17 @@ struct KeyboardSurfaceRoot: View {
|
||||
onInsert: onInsert,
|
||||
onDeleteBackward: onDeleteBackward
|
||||
)
|
||||
case .ai:
|
||||
AIKeyboardView(
|
||||
state: state,
|
||||
typing: typing,
|
||||
onInsert: onInsert
|
||||
)
|
||||
}
|
||||
}
|
||||
.animation(.easeInOut(duration: 0.15), value: state.surface)
|
||||
.onChange(of: state.surface) { _, newSurface in
|
||||
if newSurface == .voice {
|
||||
if newSurface != .typing {
|
||||
typing.leaveTypingMode()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
// AIKeyboardView.swift
|
||||
// OSGKeyboard · Keyboard Extension
|
||||
//
|
||||
// Temporary voice-to-AI surface. The latest answer remains visible while a
|
||||
// follow-up is running and is inserted only through the explicit Send action.
|
||||
|
||||
import SwiftUI
|
||||
import OSGKeyboardShared
|
||||
|
||||
struct AIKeyboardView: View {
|
||||
private enum Layout {
|
||||
static let contentHeight: CGFloat = 174
|
||||
static let actionRowHeight: CGFloat = 55
|
||||
static let actionButtonHeight: CGFloat = 50
|
||||
static let actionButtonMaxWidth: CGFloat = 150
|
||||
static let statusHeight: CGFloat = 20
|
||||
}
|
||||
|
||||
@Environment(\.colorScheme) private var colorScheme
|
||||
@ObservedObject var state: KeyboardState
|
||||
@ObservedObject var typing: TypingSessionController
|
||||
let onInsert: (String) -> Void
|
||||
|
||||
private var palette: ThemePalette {
|
||||
colorScheme == .dark ? Palette.dark : Palette.light
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
VStack(spacing: 0) {
|
||||
topBar.frame(height: KeyboardTopBarMetrics.height)
|
||||
answerArea.frame(height: resolvedAnswerHeight)
|
||||
actionRow.frame(height: Layout.actionRowHeight)
|
||||
}
|
||||
.frame(maxWidth: KeyboardChromeLayout.voiceContentMaxWidth)
|
||||
.padding(.vertical, 4)
|
||||
.padding(.horizontal, KeyboardChromeLayout.horizontalInset)
|
||||
.frame(maxWidth: .infinity)
|
||||
.frame(height: resolvedHeight)
|
||||
.environment(\.themePalette, palette)
|
||||
}
|
||||
|
||||
private var resolvedHeight: CGFloat {
|
||||
TypingSurfaceMetrics.contentHeight(
|
||||
isIPad: state.usesIPadLayoutMetrics,
|
||||
width: state.layoutWidth
|
||||
)
|
||||
}
|
||||
|
||||
private var resolvedAnswerHeight: CGFloat {
|
||||
max(
|
||||
Layout.contentHeight,
|
||||
resolvedHeight
|
||||
- KeyboardTopBarMetrics.height
|
||||
- Layout.actionRowHeight
|
||||
- 8
|
||||
)
|
||||
}
|
||||
|
||||
private var topBar: some View {
|
||||
HStack(spacing: Spacing.xs) {
|
||||
KeyboardBrandLogo(action: state.openSettings)
|
||||
Spacer(minLength: 0)
|
||||
if state.canCancelAIInput {
|
||||
KeyboardCancelButton(
|
||||
action: state.cancelAIInput,
|
||||
accessibilityLabel: ExtL10n.text("keyboard.ai.cancel"),
|
||||
accessibilityHint: ExtL10n.text("keyboard.ai.cancelHint")
|
||||
)
|
||||
} else {
|
||||
KeyboardTopControls(
|
||||
state: state,
|
||||
typing: typing,
|
||||
palette: palette,
|
||||
onInsert: onInsert
|
||||
)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal, KeyboardTopBarMetrics.nestedHorizontalInset)
|
||||
}
|
||||
|
||||
private var answerArea: some View {
|
||||
ZStack(alignment: .bottom) {
|
||||
if showsPlaceholder {
|
||||
// Empty-state tip: geometric center of the answer plane.
|
||||
Text(ExtL10n.string("keyboard.ai.placeholder"))
|
||||
.font(TypeStyle.body)
|
||||
.foregroundStyle(palette.textTertiary)
|
||||
.multilineTextAlignment(.center)
|
||||
.padding(.horizontal, Spacing.md)
|
||||
.frame(maxWidth: .infinity, maxHeight: .infinity)
|
||||
} else {
|
||||
ScrollViewReader { proxy in
|
||||
ScrollView(.vertical) {
|
||||
Group {
|
||||
if let draft = state.aiSession.draftAnswerText,
|
||||
!draft.isEmpty {
|
||||
Text(draft)
|
||||
.foregroundStyle(palette.textPrimary)
|
||||
.id("ai-draft")
|
||||
} else if let answer = state.aiSession.answer {
|
||||
Text(answer.text)
|
||||
.foregroundStyle(palette.textPrimary)
|
||||
.id(answer.id)
|
||||
}
|
||||
}
|
||||
.font(TypeStyle.body)
|
||||
.frame(maxWidth: .infinity, alignment: .topLeading)
|
||||
.padding(.horizontal, Spacing.md)
|
||||
.padding(.top, Spacing.sm)
|
||||
.padding(.bottom, Layout.statusHeight + Spacing.sm)
|
||||
}
|
||||
.scrollIndicators(.visible)
|
||||
.onChange(of: state.aiSession.answer?.id) { _, answerID in
|
||||
guard let answerID else { return }
|
||||
proxy.scrollTo(answerID, anchor: .top)
|
||||
}
|
||||
.onChange(of: state.aiSession.draftAnswerText) { _, draft in
|
||||
guard let draft, !draft.isEmpty else { return }
|
||||
proxy.scrollTo("ai-draft", anchor: .bottom)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
statusLine
|
||||
.frame(height: Layout.statusHeight)
|
||||
.padding(.horizontal, Spacing.md)
|
||||
}
|
||||
}
|
||||
|
||||
/// No draft/answer yet — show the centered mic guidance instead of a scroll body.
|
||||
private var showsPlaceholder: Bool {
|
||||
let hasDraft = !(state.aiSession.draftAnswerText?.isEmpty ?? true)
|
||||
return !hasDraft && state.aiSession.answer == nil
|
||||
}
|
||||
|
||||
private var statusLine: some View {
|
||||
// Loading spinner lives on the mic button only — avoid a second
|
||||
// ProgressView beside the status / draft caption.
|
||||
Text(statusText)
|
||||
.font(TypeStyle.caption)
|
||||
.foregroundStyle(
|
||||
state.aiSession.phase == .failed
|
||||
? palette.warning
|
||||
: palette.textSecondary
|
||||
)
|
||||
.lineLimit(1)
|
||||
.truncationMode(.head)
|
||||
.frame(maxWidth: .infinity, alignment: .center)
|
||||
}
|
||||
|
||||
private var actionRow: some View {
|
||||
HStack(spacing: Spacing.sm) {
|
||||
aiMicrophoneButton
|
||||
sendButton
|
||||
}
|
||||
.frame(maxWidth: .infinity)
|
||||
}
|
||||
|
||||
private var aiMicrophoneButton: some View {
|
||||
Button(action: state.tapAIMic) {
|
||||
ZStack {
|
||||
Capsule().fill(palette.accent)
|
||||
if state.aiSession.phase == .listening {
|
||||
Capsule()
|
||||
.stroke(Color.white.opacity(0.28), lineWidth: 1.5)
|
||||
.scaleEffect(1 + min(max(state.level, 0), 1) * 0.08)
|
||||
.animation(Motion.soft, value: state.level)
|
||||
}
|
||||
microphoneContent
|
||||
}
|
||||
.frame(
|
||||
maxWidth: Layout.actionButtonMaxWidth,
|
||||
minHeight: Layout.actionButtonHeight,
|
||||
maxHeight: Layout.actionButtonHeight
|
||||
)
|
||||
.contentShape(Capsule())
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.disabled(microphoneDisabled)
|
||||
.accessibilityLabel(ExtL10n.text(microphoneAccessibilityKey))
|
||||
}
|
||||
|
||||
@ViewBuilder
|
||||
private var microphoneContent: some View {
|
||||
switch state.aiSession.phase {
|
||||
case .listening:
|
||||
WaveformView(
|
||||
level: state.level,
|
||||
barCount: 7,
|
||||
color: .white,
|
||||
active: true
|
||||
)
|
||||
.frame(width: 35, height: 22)
|
||||
.clipped()
|
||||
case .preparing, .recognizing, .generating:
|
||||
ProgressView().tint(.white)
|
||||
case .inactive, .idle, .ready, .awaitingSend, .inserted, .sent, .failed:
|
||||
Image(systemName: "mic.fill")
|
||||
.font(.system(size: 21, weight: .semibold))
|
||||
.foregroundStyle(.white)
|
||||
}
|
||||
}
|
||||
|
||||
private var sendButton: some View {
|
||||
Button(action: state.sendAIAnswer) {
|
||||
HStack(spacing: Spacing.xs) {
|
||||
Image(systemName: answerActionSystemName)
|
||||
Text(answerActionTitle)
|
||||
}
|
||||
.font(.system(size: 16, weight: .semibold))
|
||||
.foregroundStyle(answerActionForeground)
|
||||
.frame(
|
||||
maxWidth: Layout.actionButtonMaxWidth,
|
||||
minHeight: Layout.actionButtonHeight,
|
||||
maxHeight: Layout.actionButtonHeight
|
||||
)
|
||||
.background(
|
||||
answerActionFill,
|
||||
in: Capsule()
|
||||
)
|
||||
.overlay(Capsule().stroke(answerActionBorder, lineWidth: 0.5))
|
||||
.contentShape(Capsule())
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.disabled(!state.aiSession.canPerformAnswerAction)
|
||||
.accessibilityLabel(Text(answerActionTitle))
|
||||
.accessibilityHint(ExtL10n.text("keyboard.ai.sendA11y"))
|
||||
}
|
||||
|
||||
private var answerActionTitle: String {
|
||||
switch state.aiSession.phase {
|
||||
case .awaitingSend:
|
||||
return ExtL10n.string("common.send")
|
||||
case .inserted:
|
||||
return ExtL10n.string("keyboard.ai.inserted")
|
||||
case .sent:
|
||||
return ExtL10n.string("keyboard.ai.sent")
|
||||
case .inactive, .idle, .preparing, .listening, .recognizing,
|
||||
.generating, .ready, .failed:
|
||||
return ExtL10n.string("keyboard.ai.insert")
|
||||
}
|
||||
}
|
||||
|
||||
private var answerActionSystemName: String {
|
||||
switch state.aiSession.phase {
|
||||
case .awaitingSend:
|
||||
return "paperplane.fill"
|
||||
case .inserted, .sent:
|
||||
return "checkmark"
|
||||
case .inactive, .idle, .preparing, .listening, .recognizing,
|
||||
.generating, .ready, .failed:
|
||||
return "plus"
|
||||
}
|
||||
}
|
||||
|
||||
private var answerActionFill: Color {
|
||||
guard state.aiSession.canPerformAnswerAction else {
|
||||
return palette.surfaceElevated
|
||||
}
|
||||
return state.aiSession.canSend
|
||||
? palette.accent
|
||||
: NativeKeyboardKeyColors.fill(for: colorScheme)
|
||||
}
|
||||
|
||||
private var answerActionForeground: Color {
|
||||
guard state.aiSession.canPerformAnswerAction else {
|
||||
return palette.textTertiary
|
||||
}
|
||||
return state.aiSession.canSend
|
||||
? .white
|
||||
: NativeKeyboardKeyColors.text(for: colorScheme)
|
||||
}
|
||||
|
||||
private var answerActionBorder: Color {
|
||||
guard state.aiSession.canSend else {
|
||||
return palette.divider
|
||||
}
|
||||
return Color.black.opacity(colorScheme == .dark ? 0.10 : 0.08)
|
||||
}
|
||||
|
||||
private var microphoneDisabled: Bool {
|
||||
switch state.aiSession.phase {
|
||||
case .preparing, .recognizing, .generating:
|
||||
return true
|
||||
case .inactive, .idle, .listening, .ready, .awaitingSend,
|
||||
.inserted, .sent, .failed:
|
||||
return state.micDisabled || !state.aiServiceAvailable
|
||||
}
|
||||
}
|
||||
|
||||
private var statusText: String {
|
||||
if let error = state.aiSession.errorMessage, !error.isEmpty {
|
||||
return error
|
||||
}
|
||||
if !state.aiServiceAvailable {
|
||||
return ExtL10n.string("keyboard.ai.error.missingAPIKey")
|
||||
}
|
||||
switch state.aiSession.phase {
|
||||
case .inactive, .idle, .ready, .awaitingSend, .inserted, .sent:
|
||||
return ""
|
||||
case .preparing:
|
||||
return ExtL10n.string("keyboard.placeholder.preparing")
|
||||
case .listening:
|
||||
return state.aiSession.transcript.isEmpty
|
||||
? ExtL10n.string("keyboard.ai.listening")
|
||||
: state.aiSession.transcript
|
||||
case .recognizing:
|
||||
return state.aiSession.transcript.isEmpty
|
||||
? ExtL10n.string("keyboard.ai.recognizing")
|
||||
: state.aiSession.transcript
|
||||
case .generating:
|
||||
if let draft = state.aiSession.draftAnswerText, !draft.isEmpty {
|
||||
return ExtL10n.string("keyboard.ai.generating")
|
||||
}
|
||||
return state.aiSession.transcript.isEmpty
|
||||
? ExtL10n.string("keyboard.ai.thinking")
|
||||
: state.aiSession.transcript
|
||||
case .failed:
|
||||
return ExtL10n.string("keyboard.ai.error.requestFailed")
|
||||
}
|
||||
}
|
||||
|
||||
private var microphoneAccessibilityKey: String {
|
||||
state.aiSession.phase == .listening
|
||||
? "keyboard.ai.stopA11y"
|
||||
: "keyboard.ai.startA11y"
|
||||
}
|
||||
}
|
||||
@@ -613,9 +613,16 @@ private struct TranscriptLine: View {
|
||||
Group {
|
||||
switch micVoiceAvailability {
|
||||
case .ready:
|
||||
ExtL10n.text("keyboard.placeholder.idle")
|
||||
// Soft tip when polish key is missing but local ASR can still run.
|
||||
if !micDisabledHint.isEmpty {
|
||||
Text(micDisabledHint)
|
||||
} else {
|
||||
ExtL10n.text("keyboard.placeholder.idle")
|
||||
}
|
||||
case .unavailable(.missingAPIKey):
|
||||
Text(micDisabledHint)
|
||||
Text(micDisabledHint.isEmpty
|
||||
? ExtL10n.string("keyboard.mic.disabled.missingApiKey")
|
||||
: micDisabledHint)
|
||||
case .unavailable(.hostNotReady):
|
||||
ExtL10n.text("keyboard.placeholder.idle")
|
||||
case .unavailable(.preparingSession):
|
||||
@@ -631,7 +638,11 @@ private struct TranscriptLine: View {
|
||||
}
|
||||
}
|
||||
.font(TypeStyle.caption)
|
||||
.foregroundStyle(isWarning ? palette.warning : palette.textTertiary)
|
||||
.foregroundStyle(
|
||||
(isWarning || (micVoiceAvailability.isReady && !micDisabledHint.isEmpty))
|
||||
? palette.warning
|
||||
: palette.textTertiary
|
||||
)
|
||||
.lineLimit(1)
|
||||
.truncationMode(.tail)
|
||||
}
|
||||
|
||||
@@ -73,12 +73,14 @@ struct KeyboardCancelButton: View {
|
||||
}
|
||||
|
||||
private enum KeyboardInputTab: CaseIterable {
|
||||
case ai
|
||||
case voice
|
||||
case chinese
|
||||
case english
|
||||
|
||||
var title: String {
|
||||
switch self {
|
||||
case .ai: return "AI"
|
||||
case .voice: return "语音"
|
||||
case .chinese: return "中文"
|
||||
case .english: return "EN"
|
||||
@@ -107,7 +109,10 @@ struct KeyboardTopControls: View {
|
||||
.foregroundStyle(
|
||||
isSelected(tab) ? palette.textPrimary : palette.textSecondary
|
||||
)
|
||||
.frame(width: tab == .english ? 34 : 42, height: 30)
|
||||
.frame(
|
||||
width: tab == .english || tab == .ai ? 34 : 42,
|
||||
height: 30
|
||||
)
|
||||
.background {
|
||||
if isSelected(tab) {
|
||||
Capsule()
|
||||
@@ -163,6 +168,8 @@ struct KeyboardTopControls: View {
|
||||
|
||||
private func isSelected(_ tab: KeyboardInputTab) -> Bool {
|
||||
switch tab {
|
||||
case .ai:
|
||||
return state.surface == .ai
|
||||
case .voice:
|
||||
return state.surface == .voice
|
||||
case .chinese:
|
||||
@@ -174,6 +181,8 @@ struct KeyboardTopControls: View {
|
||||
|
||||
private func select(_ tab: KeyboardInputTab) {
|
||||
switch tab {
|
||||
case .ai:
|
||||
state.setSurface(.ai)
|
||||
case .voice:
|
||||
state.setSurface(.voice)
|
||||
case .chinese:
|
||||
@@ -202,6 +211,7 @@ struct KeyboardTopControls: View {
|
||||
|
||||
private func accessibilityLabel(for tab: KeyboardInputTab) -> String {
|
||||
switch tab {
|
||||
case .ai: return "切换到 AI 问答"
|
||||
case .voice: return "切换到语音输入"
|
||||
case .chinese: return "切换到中文输入"
|
||||
case .english: return "切换到英文输入"
|
||||
|
||||
@@ -11,9 +11,9 @@
|
||||
"onboarding.enable.step4" = "Allow Full Access is required for the microphone and LLM calls.";
|
||||
"onboarding.enable.openSettings" = "Open iOS Settings";
|
||||
"onboarding.api.title" = "Choose engine";
|
||||
"onboarding.api.subtitle" = "Local needs no API key; Cloud polishes your text via LLM.";
|
||||
"onboarding.api.localReady.title" = "No API key needed";
|
||||
"onboarding.api.localReady.body" = "Local recognition is ready to use. Tap Done to start.";
|
||||
"onboarding.api.subtitle" = "Local ASR needs no key; add an API key to enable AI polish.";
|
||||
"onboarding.api.localReady.title" = "Local ASR is ready";
|
||||
"onboarding.api.localReady.body" = "Recognition works on-device. Add an API key in Settings for AI polish.";
|
||||
|
||||
/* Common navigation */
|
||||
"common.back" = "Back";
|
||||
@@ -168,7 +168,8 @@
|
||||
"keyboard.error.manualOpenDictate" = "System blocked the jump. Open OSGKeyboard to record, then return.";
|
||||
"keyboard.error.llm.noApiKey" = "API key missing · configure it in the main app";
|
||||
"keyboard.mic.disabled.missingApiKey" = "Fill in API key in Settings first";
|
||||
"keyboard.error.llm.localPolishUnavailable" = "Built-in polish unavailable · inserted raw text";
|
||||
"keyboard.mic.hint.missingPolishApiKey" = "Add an API key in Settings to enable polish";
|
||||
"keyboard.error.llm.localPolishUnavailable" = "API key missing · inserted raw text";
|
||||
"keyboard.error.llm.unauthorized" = "Invalid API key (401) · check main app settings";
|
||||
"keyboard.error.llm.rateLimited" = "Rate limited (429) · try again later";
|
||||
|
||||
@@ -261,3 +262,25 @@
|
||||
"keyboard.edit.apply" = "Apply edit";
|
||||
"keyboard.edit.append" = "Insert at cursor";
|
||||
"keyboard.edit.stop" = "Finish editing instruction";
|
||||
|
||||
/* AI question mode */
|
||||
"keyboard.ai.placeholder" = "Tap the microphone to ask AI";
|
||||
"keyboard.ai.hint" = "Insert the AI answer, then tap Send";
|
||||
"keyboard.ai.listening" = "Listening…";
|
||||
"keyboard.ai.recognizing" = "Recognizing your question…";
|
||||
"keyboard.ai.generating" = "AI is answering…";
|
||||
"keyboard.ai.thinking" = "AI is thinking…";
|
||||
"keyboard.ai.send" = "Send";
|
||||
"keyboard.ai.insert" = "Insert";
|
||||
"keyboard.ai.inserted" = "Inserted";
|
||||
"keyboard.ai.sent" = "Sent";
|
||||
"keyboard.ai.cancel" = "Cancel AI question";
|
||||
"keyboard.ai.cancelHint" = "Cancel the current recording, recognition, or AI request.";
|
||||
"keyboard.ai.sendA11y" = "Tap once to insert the AI answer, then tap again to send in supported fields";
|
||||
"keyboard.ai.startA11y" = "Start asking AI";
|
||||
"keyboard.ai.stopA11y" = "Finish the question and send it to AI";
|
||||
"keyboard.ai.error.missingAPIKey" = "Configure an AI service in the main app first";
|
||||
"keyboard.ai.error.pipelineBusy" = "Voice input is busy. Try again shortly";
|
||||
"keyboard.ai.error.startTimeout" = "Microphone startup timed out. Try again";
|
||||
"keyboard.ai.error.requestTimeout" = "AI response timed out. Try again";
|
||||
"keyboard.ai.error.requestFailed" = "AI response failed. Try again";
|
||||
|
||||
@@ -11,9 +11,9 @@
|
||||
"onboarding.enable.step4" = "允许完全访问是麦克风和网络调用的前提。";
|
||||
"onboarding.enable.openSettings" = "打开 iOS 设置";
|
||||
"onboarding.api.title" = "选择引擎";
|
||||
"onboarding.api.subtitle" = "Local 不需要 API Key;Cloud 用 LLM 润色文字。";
|
||||
"onboarding.api.localReady.title" = "无需配置 API Key";
|
||||
"onboarding.api.localReady.body" = "本地识别直接可用,按 Done 即可开始使用。";
|
||||
"onboarding.api.subtitle" = "本地识别无需 Key;填写 API Key 后可开启 AI 润色。";
|
||||
"onboarding.api.localReady.title" = "本地识别已就绪";
|
||||
"onboarding.api.localReady.body" = "识别在端侧完成。请在设置中填写 API Key 以开启 AI 润色。";
|
||||
|
||||
/* Common navigation */
|
||||
"common.back" = "返回";
|
||||
@@ -168,7 +168,8 @@
|
||||
"keyboard.error.manualOpenDictate" = "系统拒绝了跳转,请手动打开 OSGKeyboard 录音";
|
||||
"keyboard.error.llm.noApiKey" = "未配置 API Key · 请在主 App 设置中填写";
|
||||
"keyboard.mic.disabled.missingApiKey" = "请先在设置中填写 API Key";
|
||||
"keyboard.error.llm.localPolishUnavailable" = "内置润色不可用 · 已插入原始文本";
|
||||
"keyboard.mic.hint.missingPolishApiKey" = "请先在设置中填写 API Key,才能润色";
|
||||
"keyboard.error.llm.localPolishUnavailable" = "未填写 API Key · 已插入原始文本";
|
||||
"keyboard.error.llm.unauthorized" = "API Key 无效 (401) · 请检查主 App 设置";
|
||||
"keyboard.error.llm.rateLimited" = "API 限流 (429) · 请稍后再试";
|
||||
|
||||
@@ -261,3 +262,25 @@
|
||||
"keyboard.edit.apply" = "应用编辑";
|
||||
"keyboard.edit.append" = "插入当前位置";
|
||||
"keyboard.edit.stop" = "完成编辑指令";
|
||||
|
||||
/* AI 问答模式 */
|
||||
"keyboard.ai.placeholder" = "点击麦克风向 AI 提问";
|
||||
"keyboard.ai.hint" = "先插入 AI 回答,再按发送";
|
||||
"keyboard.ai.listening" = "正在聆听…";
|
||||
"keyboard.ai.recognizing" = "正在识别问题…";
|
||||
"keyboard.ai.generating" = "AI 正在回答…";
|
||||
"keyboard.ai.thinking" = "AI 正在思考…";
|
||||
"keyboard.ai.send" = "发送";
|
||||
"keyboard.ai.insert" = "插入";
|
||||
"keyboard.ai.inserted" = "已插入";
|
||||
"keyboard.ai.sent" = "已发送";
|
||||
"keyboard.ai.cancel" = "取消 AI 问答";
|
||||
"keyboard.ai.cancelHint" = "取消当前录音、识别或 AI 请求。";
|
||||
"keyboard.ai.sendA11y" = "首次点击插入 AI 回答;在支持发送的输入框中再次点击发送";
|
||||
"keyboard.ai.startA11y" = "开始向 AI 提问";
|
||||
"keyboard.ai.stopA11y" = "结束提问并发送给 AI";
|
||||
"keyboard.ai.error.missingAPIKey" = "请先在主 App 配置可用的 AI 服务";
|
||||
"keyboard.ai.error.pipelineBusy" = "语音服务正忙,请稍后重试";
|
||||
"keyboard.ai.error.startTimeout" = "麦克风启动超时,请重试";
|
||||
"keyboard.ai.error.requestTimeout" = "AI 回答超时,请重试";
|
||||
"keyboard.ai.error.requestFailed" = "AI 回答失败,请重试";
|
||||
|
||||
Reference in New Issue
Block a user