Rocky
fb97ec2937
Merge dictation and AI controls into one assistant surface, preserve safe insertion and clipboard actions, and align settings and tests with the new flow.
418 lines
16 KiB
Swift
418 lines
16 KiB
Swift
// AIKeyboardCoordinator.swift
|
|
// OSGKeyboard · Keyboard Extension
|
|
//
|
|
// Owns the temporary AI-mode UI state. Audio, ASR, and LLM work remain in the
|
|
// shared Flow transport and host app; this coordinator never performs network
|
|
// work and never inserts an answer before explicit user confirmation.
|
|
|
|
import Foundation
|
|
import OSGKeyboardShared
|
|
|
|
@MainActor
|
|
final class AIKeyboardCoordinator {
|
|
private let state: KeyboardState
|
|
private let flow: KeyboardFlowCoordinator
|
|
private let insertAnswer: (AIAnswer) -> Bool
|
|
private let performReturn: () -> Void
|
|
private let captureInsertionFingerprint: () -> String?
|
|
private var requestInsertionFingerprint: String?
|
|
private var conversationInsertionFingerprint: String?
|
|
private var hasConversationInsertionTarget = false
|
|
|
|
init(
|
|
state: KeyboardState,
|
|
flow: KeyboardFlowCoordinator,
|
|
insertAnswer: @escaping (AIAnswer) -> Bool,
|
|
performReturn: @escaping () -> Void,
|
|
captureInsertionFingerprint: @escaping () -> String?
|
|
) {
|
|
self.state = state
|
|
self.flow = flow
|
|
self.insertAnswer = insertAnswer
|
|
self.performReturn = performReturn
|
|
self.captureInsertionFingerprint = captureInsertionFingerprint
|
|
}
|
|
|
|
func beginNewPresentation() {
|
|
endConversationIfNeeded()
|
|
state.aiSession.enter()
|
|
requestInsertionFingerprint = nil
|
|
conversationInsertionFingerprint = nil
|
|
hasConversationInsertionTarget = false
|
|
}
|
|
|
|
func enterIfNeeded() {
|
|
guard !state.aiSession.isActive else { return }
|
|
state.aiSession.enter()
|
|
}
|
|
|
|
func leave() {
|
|
if state.aiSession.isBusy {
|
|
flow.cancelAIRecording()
|
|
}
|
|
endConversationIfNeeded()
|
|
state.aiSession.leave()
|
|
requestInsertionFingerprint = nil
|
|
conversationInsertionFingerprint = nil
|
|
hasConversationInsertionTarget = false
|
|
}
|
|
|
|
func toggleMicrophone() {
|
|
switch state.aiSession.phase {
|
|
case .listening:
|
|
guard let utteranceID = state.aiSession.activeUtteranceID else { return }
|
|
state.aiSession.beginRecognizing(utteranceID: utteranceID)
|
|
flow.stopAIRecording()
|
|
case .idle, .awaitingSend, .inserted, .sent, .failed:
|
|
prepareConversationForRequest()
|
|
guard let conversationID = state.aiSession.conversationID else { return }
|
|
let disposition = flow.beginAIRecording(conversationID: conversationID)
|
|
if case .rejected(let rejection) = disposition {
|
|
state.aiSession.fail(message(for: rejection), utteranceID: nil)
|
|
}
|
|
case .inactive:
|
|
prepareConversationForRequest()
|
|
toggleMicrophone()
|
|
case .preparing, .recognizing, .generating, .ready:
|
|
break
|
|
}
|
|
}
|
|
|
|
/// Tap a clipboard skill chip: same fail-closed material path as hint cards.
|
|
func submitClipboardSkill(_ skill: AIClipboardSkill) {
|
|
guard canAcceptIdleSubmit else { return }
|
|
prepareConversationForRequest()
|
|
let material = ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
|
|
if skill.kind == .export {
|
|
state.pendingClipboardSkillID = skill.id
|
|
state.pendingClipboardSkillSource = material
|
|
} else {
|
|
clearPendingExportSkill()
|
|
}
|
|
var instruction = AIClipboardSkillCatalog.instruction(
|
|
for: skill,
|
|
locale: AIHintLocaleResolver.packLocale(),
|
|
translationTargetLocaleId: state.translationTargetLocaleId
|
|
)
|
|
if skill.kind == .export {
|
|
instruction += "\nPreserve the source language, addresses, names, and proper nouns."
|
|
}
|
|
AIAgentShortcutRun.trace("keyboard.submit skill=\(skill.id) kind=\(skill.kind)")
|
|
if let material {
|
|
AIAgentShortcutRun.traceBody("keyboard.clipboard", material)
|
|
} else {
|
|
AIAgentShortcutRun.trace("keyboard.clipboard missing")
|
|
}
|
|
let resolution = AIClipboardPrompt.resolve(
|
|
instruction: instruction,
|
|
material: material
|
|
)
|
|
if case .materialUnavailable = resolution {
|
|
AIAgentShortcutRun.trace("keyboard.submit rejected clipboardUnavailable skill=\(skill.id)")
|
|
}
|
|
submitResolvedPrompt(resolution, thinkingEnabled: skill.thinkingEnabled)
|
|
}
|
|
|
|
/// Tap an idle hint card: resolve its material, skip the mic, ask the host.
|
|
func submitHintCard(_ card: AIHintCard) {
|
|
guard canAcceptIdleSubmit else { return }
|
|
prepareConversationForRequest()
|
|
let resolution = AIHintPool.resolvePrompt(
|
|
for: card,
|
|
clipboardText: ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
|
|
)
|
|
submitResolvedPrompt(resolution)
|
|
}
|
|
|
|
private var canAcceptIdleSubmit: Bool {
|
|
switch state.aiSession.phase {
|
|
case .inactive, .idle, .awaitingSend, .inserted, .sent, .failed:
|
|
return true
|
|
case .preparing, .listening, .recognizing, .generating,
|
|
.ready:
|
|
return false
|
|
}
|
|
}
|
|
|
|
private func submitResolvedPrompt(
|
|
_ resolution: AIClipboardPrompt.Resolution,
|
|
thinkingEnabled: Bool? = nil
|
|
) {
|
|
guard case .ready(let prompt) = resolution else {
|
|
// The clipboard window closed between rendering and this tap.
|
|
clearPendingExportSkill()
|
|
state.aiSession.fail(
|
|
ExtL10n.string("keyboard.ai.error.clipboardUnavailable"),
|
|
utteranceID: nil
|
|
)
|
|
return
|
|
}
|
|
guard let conversationID = state.aiSession.conversationID else {
|
|
clearPendingExportSkill()
|
|
return
|
|
}
|
|
let disposition = flow.submitAIQuestion(
|
|
text: prompt,
|
|
conversationID: conversationID,
|
|
thinkingEnabled: thinkingEnabled
|
|
)
|
|
if case .rejected(let rejection) = disposition {
|
|
clearPendingExportSkill()
|
|
state.aiSession.fail(message(for: rejection), utteranceID: nil)
|
|
}
|
|
}
|
|
|
|
func cancel() {
|
|
guard state.aiSession.isBusy else { return }
|
|
clearPendingExportSkill()
|
|
requestInsertionFingerprint = nil
|
|
flow.cancelAIRecording()
|
|
state.aiSession.cancelCurrentWork()
|
|
}
|
|
|
|
func confirmPendingAnswer() {
|
|
if state.aiSession.canInsert, let answer = state.aiSession.answer {
|
|
guard insertAnswer(answer) else { return }
|
|
state.aiSession.markAnswerInserted(
|
|
offersSend: state.returnKeyRole.usesActionFill
|
|
)
|
|
conversationInsertionFingerprint = captureInsertionFingerprint()
|
|
hasConversationInsertionTarget = true
|
|
}
|
|
}
|
|
|
|
func discardPendingAnswer() {
|
|
state.aiSession.discardReadyAnswer()
|
|
requestInsertionFingerprint = nil
|
|
}
|
|
|
|
func sendCurrentFieldAction() {
|
|
guard state.assistantSendAvailable else { return }
|
|
state.assistantSendAvailable = false
|
|
if state.aiSession.canSend {
|
|
state.aiSession.markAnswerSent()
|
|
}
|
|
// Let the host consume the inserted answer before issuing Return.
|
|
Task { @MainActor [weak self] in
|
|
await Task.yield()
|
|
self?.performReturn()
|
|
}
|
|
}
|
|
|
|
func utterancePrepared(_ utteranceID: UUID) {
|
|
state.aiSession.beginPreparing(utteranceID: utteranceID)
|
|
}
|
|
|
|
func recordingStarted(_ utteranceID: UUID) {
|
|
state.aiSession.beginListening(utteranceID: utteranceID)
|
|
}
|
|
|
|
func recognitionStarted(_ utteranceID: UUID) {
|
|
state.aiSession.beginRecognizing(utteranceID: utteranceID)
|
|
}
|
|
|
|
func generatingStarted(_ utteranceID: UUID) {
|
|
state.aiSession.beginGenerating(question: "", utteranceID: utteranceID)
|
|
}
|
|
|
|
func receiveTranscript(
|
|
_ transcript: String,
|
|
utteranceID: UUID,
|
|
status: FlowResult.Status
|
|
) {
|
|
if AIClipboardPrompt.isInternalPrompt(transcript) {
|
|
if status == .rawReady {
|
|
state.aiSession.beginGenerating(question: "", utteranceID: utteranceID)
|
|
}
|
|
return
|
|
}
|
|
state.aiSession.updateTranscript(transcript, utteranceID: utteranceID)
|
|
if status == .rawReady {
|
|
state.aiSession.beginGenerating(
|
|
question: transcript,
|
|
utteranceID: utteranceID
|
|
)
|
|
}
|
|
}
|
|
|
|
func receivePartialAnswer(_ draft: String, utteranceID: UUID) {
|
|
if isPendingExportSkill { return }
|
|
state.aiSession.receivePartialAnswer(draft, utteranceID: utteranceID)
|
|
}
|
|
|
|
func receive(result: FlowResult) {
|
|
guard result.resolvedUtteranceMode == .aiQuestion else {
|
|
return
|
|
}
|
|
if isPendingExportSkill {
|
|
guard result.aiConversationID == state.aiSession.conversationID else { return }
|
|
finishExportSkill(answer: result.text ?? "")
|
|
return
|
|
}
|
|
guard result.aiConversationID == state.aiSession.conversationID,
|
|
let answer = result.text,
|
|
!answer.isEmpty else {
|
|
state.aiSession.fail(
|
|
ExtL10n.string("keyboard.ai.error.requestFailed"),
|
|
utteranceID: result.utteranceId
|
|
)
|
|
return
|
|
}
|
|
state.aiSession.receiveAnswer(answer, utteranceID: result.utteranceId)
|
|
defer { requestInsertionFingerprint = nil }
|
|
guard state.aiSession.canInsert,
|
|
let expected = requestInsertionFingerprint,
|
|
captureInsertionFingerprint() == expected,
|
|
let answer = state.aiSession.answer,
|
|
insertAnswer(answer) else {
|
|
// Keep `.ready`: the unified UI presents an explicit Insert / Discard
|
|
// fallback when the field or caret changed during generation.
|
|
return
|
|
}
|
|
state.aiSession.markAnswerInserted(
|
|
offersSend: state.returnKeyRole.usesActionFill
|
|
)
|
|
conversationInsertionFingerprint = captureInsertionFingerprint()
|
|
hasConversationInsertionTarget = true
|
|
}
|
|
|
|
func fail(_ message: String, utteranceID: UUID?) {
|
|
clearPendingExportSkill()
|
|
requestInsertionFingerprint = nil
|
|
state.aiSession.fail(message, utteranceID: utteranceID)
|
|
}
|
|
|
|
/// A global output-language change starts a clean conversation so retained
|
|
/// turns cannot override the newly selected language policy.
|
|
func resetConversationForConfigurationChange() {
|
|
guard state.aiSession.isActive else { return }
|
|
beginNewPresentation()
|
|
}
|
|
|
|
private func endConversationIfNeeded() {
|
|
clearPendingExportSkill()
|
|
guard let conversationID = state.aiSession.conversationID else { return }
|
|
flow.endAIConversation(conversationID)
|
|
}
|
|
|
|
private func clearPendingExportSkill() {
|
|
state.pendingClipboardSkillID = nil
|
|
state.pendingClipboardSkillSource = nil
|
|
}
|
|
|
|
private func prepareConversationForRequest() {
|
|
let currentFingerprint = captureInsertionFingerprint()
|
|
if state.aiSession.isActive,
|
|
hasConversationInsertionTarget,
|
|
conversationInsertionFingerprint != currentFingerprint {
|
|
beginNewPresentation()
|
|
} else {
|
|
enterIfNeeded()
|
|
}
|
|
conversationInsertionFingerprint = currentFingerprint
|
|
hasConversationInsertionTarget = true
|
|
requestInsertionFingerprint = currentFingerprint
|
|
}
|
|
|
|
private var isPendingExportSkill: Bool {
|
|
guard let id = state.pendingClipboardSkillID else { return false }
|
|
return resolvedSkill(id: id)?.kind == .export
|
|
}
|
|
|
|
private func resolvedSkill(id: String) -> AIClipboardSkill? {
|
|
AIClipboardSkillCatalog.skill(
|
|
id: id,
|
|
userCatalog: AppGroupStore().agentUserSkillCatalog
|
|
)
|
|
}
|
|
|
|
/// Parse an export skill. Empty → in-keyboard tip, stay in the host app.
|
|
/// Lines → hand off to the host (Shortcut, Maps, or Didi).
|
|
private func finishExportSkill(answer: String) {
|
|
let source = state.pendingClipboardSkillSource
|
|
?? ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
|
|
let skillID = state.pendingClipboardSkillID
|
|
let items: [String]
|
|
let emptyTipKey: String
|
|
switch skillID {
|
|
case AIClipboardSkillCatalog.extractEventsID:
|
|
items = AIEventExtraction.lines(from: answer, sourceClipboard: source)
|
|
emptyTipKey = "keyboard.ai.skill.noEvents"
|
|
case AIClipboardSkillCatalog.extractTodosID:
|
|
items = AITodoExtraction.items(from: answer, sourceClipboard: source)
|
|
emptyTipKey = "keyboard.ai.skill.noTodos"
|
|
case AIClipboardSkillCatalog.navigateID:
|
|
items = AIAddressExtraction.lines(from: answer, sourceClipboard: source)
|
|
emptyTipKey = "keyboard.ai.skill.noAddress"
|
|
case AIClipboardSkillCatalog.saveToNotesID:
|
|
items = AINoteExport.items(
|
|
from: answer,
|
|
sourceClipboard: source,
|
|
locale: AIHintLocaleResolver.packLocale()
|
|
)
|
|
emptyTipKey = "keyboard.ai.skill.noNote"
|
|
default:
|
|
items = AIGenericSkillExport.items(from: answer)
|
|
emptyTipKey = "keyboard.ai.skill.noExportItems"
|
|
}
|
|
AIAgentShortcutRun.traceBody("keyboard.llmRaw", answer)
|
|
AIAgentShortcutRun.trace(
|
|
"keyboard.parse items=\(items.count) clipboardChars=\(source?.count ?? 0)"
|
|
)
|
|
#if DEBUG
|
|
if !items.isEmpty {
|
|
AIAgentShortcutRun.traceBody("keyboard.parsedTitles", items.joined(separator: "\n"))
|
|
}
|
|
#endif
|
|
clearPendingExportSkill()
|
|
if state.aiSession.isBusy {
|
|
state.aiSession.cancelCurrentWork()
|
|
}
|
|
if state.aiSession.isActive {
|
|
state.aiSession.resetConversationPreservingAnswer()
|
|
}
|
|
guard !items.isEmpty else {
|
|
AIAgentShortcutRun.trace("keyboard.parse empty — skip Shortcuts")
|
|
state.skillTipText = ExtL10n.string(emptyTipKey)
|
|
return
|
|
}
|
|
guard let skillID, let skill = resolvedSkill(id: skillID) else {
|
|
AIAgentShortcutRun.trace("keyboard.parse missingSkill")
|
|
state.skillTipText = ExtL10n.string("keyboard.ai.skill.shortcutMissing")
|
|
return
|
|
}
|
|
if skill.requiresShortcut, skill.shortcutName == nil {
|
|
AIAgentShortcutRun.trace("keyboard.parse missingShortcut skill=\(skillID)")
|
|
state.skillTipText = ExtL10n.string("keyboard.ai.skill.shortcutMissing")
|
|
return
|
|
}
|
|
AIAgentShortcutRun.trace("keyboard.handoffToHost skill=\(skillID) items=\(items.count)")
|
|
let tipKey: String
|
|
switch skillID {
|
|
case AIClipboardSkillCatalog.navigateID:
|
|
tipKey = "keyboard.ai.skill.openingMaps"
|
|
default:
|
|
tipKey = "keyboard.ai.skill.runningShortcut"
|
|
}
|
|
state.skillTipText = ExtL10n.string(tipKey)
|
|
state.runClipboardExportSkill(skillID, items)
|
|
}
|
|
|
|
private func message(for rejection: FlowUtteranceStartRejection) -> String {
|
|
switch rejection {
|
|
case .onboardingIncomplete:
|
|
return ExtL10n.string("keyboard.hint.finishSetupInApp")
|
|
case .missingAPIKey:
|
|
return ExtL10n.string("keyboard.ai.error.missingAPIKey")
|
|
case .noFullAccess:
|
|
return ExtL10n.string("keyboard.error.fullAccessRequired")
|
|
case .appGroupUnavailable:
|
|
return ExtL10n.string("keyboard.error.appGroupCommunication")
|
|
case .hostUnavailable:
|
|
return ExtL10n.string("keyboard.flow.hostDisconnected")
|
|
case .pipelineBusy:
|
|
return ExtL10n.string("keyboard.ai.error.pipelineBusy")
|
|
}
|
|
}
|
|
}
|