rockyandClaude Opus 5
0023741455
Squashes the working tree that had accumulated on cloud-agent. Two strands of work, kept in one commit at the maintainer's request. Crash and stability (this session): - Subscribe to MetricKit so crash, hang, CPU, and disk-write diagnostics are recorded on device. Keyboard-extension memory kills are the most likely way a user loses the keyboard mid-sentence and iOS does not classify them as crashes, so they never reach App Store Connect. - Upload those reports to POST /v1/analytics/diagnostics from internal builds only (local Debug and TestFlight), reusing the analytics installation identifier and skipping entirely when analytics are off. App Store builds keep everything on device. - Shed the Chinese engine, English lexicon, and in-flight pipeline work at the 40 MiB and 48 MiB budget thresholds instead of waiting for a system memory warning that jetsam often preempts. - Install audio taps with the input node's live format through one shared adaptive resampler, so an audio-route change mid-recording no longer raises an uncatchable format-mismatch exception. Covers the dictation preview and the Mac recorder, which still used the pattern the Flow capture path had already moved away from. - Harden the librime bridge: never pass a nil string across a nonnull boundary when a user dictionary holds malformed UTF-8, null-check the engine API vector, and distrust a candidate count without its array. - Guard the expanded candidate grid against a list that shrinks while the collection view is still laying out against the previous count. - Resolve localized text without the App Group so the screen shown when that container is missing renders its message instead of crashing on launch, which is the one situation it exists to handle. - Gate CI on the App Group and Keychain wiring that the keyboard extension hard-depends on, with a self-test proving the check fails. Clipboard, reply, and semantics (pre-existing working tree): - Unify the reply center, refresh keyboard AI features, and add native event and reminder exporters. - Expand clipboard semantic routing with authored corpora and a product-blind holdout, plus the scripts that build and validate them. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
918 lines
38 KiB
Swift
918 lines
38 KiB
Swift
// AIKeyboardCoordinator.swift
|
|
// OSGKeyboard · Keyboard Extension
|
|
//
|
|
// Owns the temporary AI-mode UI state. Audio, ASR, and LLM work remain in the
|
|
// shared Flow transport and host app; this coordinator never performs network
|
|
// work and never inserts an answer before explicit user confirmation.
|
|
|
|
import Foundation
|
|
import OSGKeyboardShared
|
|
|
|
@MainActor
|
|
final class AIKeyboardCoordinator {
|
|
private let state: KeyboardState
|
|
private let flow: KeyboardFlowCoordinator
|
|
private let insertAnswer: (AIAnswer) -> Bool
|
|
private let performReturn: () -> Void
|
|
private let captureInsertionFingerprint: () -> String?
|
|
private let openWebURL: (URL) -> Void
|
|
private let callPhone: (String) -> Void
|
|
private let createContact: (String) -> Void
|
|
private let replyFeedbackStore: ClipboardReplyFeedbackStore
|
|
private var requestInsertionFingerprint: String?
|
|
private var conversationInsertionFingerprint: String?
|
|
private var hasConversationInsertionTarget = false
|
|
private var requestOOBEFeature: ManagedGatewayOOBEFeature?
|
|
private var requestExpectsReplyVariants = false
|
|
private var requestReplyVariantSet: AIReplyVariantSet = .generic
|
|
private var requestReplySourceText: String?
|
|
private var requestReplyFeedbackSource: String?
|
|
private var pendingReplyFeedbackRecordID: UUID?
|
|
private var pendingStructuredReplyResult = false
|
|
|
|
init(
|
|
state: KeyboardState,
|
|
flow: KeyboardFlowCoordinator,
|
|
insertAnswer: @escaping (AIAnswer) -> Bool,
|
|
performReturn: @escaping () -> Void,
|
|
captureInsertionFingerprint: @escaping () -> String?,
|
|
openWebURL: @escaping (URL) -> Void,
|
|
callPhone: @escaping (String) -> Void,
|
|
createContact: @escaping (String) -> Void,
|
|
replyFeedbackStore: ClipboardReplyFeedbackStore = .shared
|
|
) {
|
|
self.state = state
|
|
self.flow = flow
|
|
self.insertAnswer = insertAnswer
|
|
self.performReturn = performReturn
|
|
self.captureInsertionFingerprint = captureInsertionFingerprint
|
|
self.openWebURL = openWebURL
|
|
self.callPhone = callPhone
|
|
self.createContact = createContact
|
|
self.replyFeedbackStore = replyFeedbackStore
|
|
}
|
|
|
|
func beginNewPresentation() {
|
|
discardPendingReplyFeedback()
|
|
endConversationIfNeeded()
|
|
state.aiSession.enter()
|
|
requestInsertionFingerprint = nil
|
|
requestOOBEFeature = nil
|
|
requestExpectsReplyVariants = false
|
|
requestReplyVariantSet = .generic
|
|
requestReplySourceText = nil
|
|
requestReplyFeedbackSource = nil
|
|
pendingStructuredReplyResult = false
|
|
conversationInsertionFingerprint = nil
|
|
hasConversationInsertionTarget = false
|
|
}
|
|
|
|
func enterIfNeeded() {
|
|
guard !state.aiSession.isActive else { return }
|
|
state.aiSession.enter()
|
|
}
|
|
|
|
func leave() {
|
|
if state.aiSession.isBusy {
|
|
flow.cancelAIRecording()
|
|
}
|
|
discardPendingReplyFeedback()
|
|
endConversationIfNeeded()
|
|
state.aiSession.leave()
|
|
requestInsertionFingerprint = nil
|
|
requestOOBEFeature = nil
|
|
requestExpectsReplyVariants = false
|
|
requestReplyVariantSet = .generic
|
|
requestReplySourceText = nil
|
|
requestReplyFeedbackSource = nil
|
|
pendingStructuredReplyResult = false
|
|
conversationInsertionFingerprint = nil
|
|
hasConversationInsertionTarget = false
|
|
// The reply session is gone; drop any pending auto-reply surface restore
|
|
// so the next presentation's surface is chosen normally.
|
|
state.autoReplyReturnSurface = nil
|
|
state.autoResultReadOnly = false
|
|
}
|
|
|
|
func toggleMicrophone() {
|
|
switch state.aiSession.phase {
|
|
case .listening:
|
|
guard let utteranceID = state.aiSession.activeUtteranceID else { return }
|
|
state.aiSession.beginRecognizing(utteranceID: utteranceID)
|
|
flow.stopAIRecording()
|
|
case .idle, .awaitingSend, .inserted, .sent, .failed:
|
|
prepareConversationForRequest()
|
|
guard let conversationID = state.aiSession.conversationID else { return }
|
|
let oobeFeature = expectedOOBEFeature(.askAI)
|
|
requestOOBEFeature = oobeFeature
|
|
let disposition = flow.beginAIRecording(
|
|
conversationID: conversationID,
|
|
oobeFeature: oobeFeature
|
|
)
|
|
if case .rejected(let rejection) = disposition {
|
|
state.aiSession.fail(message(for: rejection), utteranceID: nil)
|
|
}
|
|
case .inactive:
|
|
prepareConversationForRequest()
|
|
toggleMicrophone()
|
|
case .preparing, .recognizing, .generating, .ready:
|
|
break
|
|
}
|
|
}
|
|
|
|
/// Tap a clipboard skill capsule: same fail-closed material path as hint cards.
|
|
func submitClipboardSkill(
|
|
_ skill: AIClipboardSkill,
|
|
replyScene: AIClipboardReplyScene? = nil
|
|
) {
|
|
// Block only while a request is actively running. A result that is merely
|
|
// displayed (.ready) is replaceable: the user tapped another skill capsule
|
|
// to switch what the AI does with this copy, so clear it and continue.
|
|
guard !state.aiSession.isBusy else { return }
|
|
if state.aiSession.phase == .ready {
|
|
discardPendingReplyFeedback()
|
|
state.aiSession.discardReadyAnswer()
|
|
}
|
|
guard !skill.requiresShortcut
|
|
|| state.confirmedClipboardShortcutIDs.contains(skill.id) else {
|
|
state.skillTipText = ExtL10n.string("keyboard.ai.skill.shortcutMissing")
|
|
return
|
|
}
|
|
let oobeFeature = oobeFeature(for: skill)
|
|
let material: String?
|
|
if let oobeFeature {
|
|
// OOBE clipboard lessons are intentionally isolated from the
|
|
// user's real clipboard history.
|
|
material = oobeMaterial(for: oobeFeature)
|
|
} else if state.oobePracticeSession != nil {
|
|
material = nil
|
|
} else {
|
|
material = ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
|
|
}
|
|
if skill.id == AIClipboardSkillCatalog.openLinkID {
|
|
guard let material,
|
|
let url = ClipboardWebLinkResolver.singleWebURL(in: material) else {
|
|
state.skillTipText = ExtL10n.string("keyboard.ai.error.clipboardUnavailable")
|
|
return
|
|
}
|
|
openWebURL(url)
|
|
return
|
|
}
|
|
if skill.id == AIClipboardSkillCatalog.callPhoneID {
|
|
guard let material,
|
|
let phoneNumber = AIPhoneNumberResolver.singlePhoneNumber(in: material) else {
|
|
state.skillTipText = ExtL10n.string("keyboard.ai.error.clipboardUnavailable")
|
|
return
|
|
}
|
|
callPhone(phoneNumber)
|
|
return
|
|
}
|
|
if skill.id == AIClipboardSkillCatalog.createContactID {
|
|
guard let material,
|
|
let phoneNumber = AIPhoneNumberResolver.singlePhoneNumber(in: material) else {
|
|
state.skillTipText = ExtL10n.string("keyboard.ai.error.clipboardUnavailable")
|
|
return
|
|
}
|
|
createContact(phoneNumber)
|
|
return
|
|
}
|
|
|
|
prepareConversationForRequest()
|
|
// Remember which skill this result came from so the result panel can
|
|
// offer the *other* applicable skills without repeating this one.
|
|
state.activeClipboardSkillID = skill.id
|
|
requestReplyFeedbackSource = skill.id == AIClipboardSkillCatalog.replyID
|
|
&& oobeFeature == nil
|
|
? material
|
|
: nil
|
|
if skill.kind == .export {
|
|
state.pendingClipboardSkillID = skill.id
|
|
state.pendingClipboardSkillSource = material
|
|
} else {
|
|
clearPendingExportSkill()
|
|
}
|
|
var instruction = AIClipboardSkillCatalog.instruction(
|
|
for: skill,
|
|
locale: AIHintLocaleResolver.packLocale(),
|
|
translationTargetLocaleId: state.translationTargetLocaleId,
|
|
replyStyle: state.clipboardReplyStyle,
|
|
replyScene: replyScene
|
|
)
|
|
let replyVariantSet = AIReplyVariantSet.resolve(scene: replyScene)
|
|
let expectsReplyVariants = skill.id == AIClipboardSkillCatalog.replyID
|
|
&& AIReplyVariantSet.shouldGenerate(
|
|
multipleRepliesEnabled: state.multipleReplyVariantsEnabled,
|
|
scene: replyScene
|
|
)
|
|
requestReplyVariantSet = expectsReplyVariants ? replyVariantSet : .generic
|
|
requestReplySourceText = expectsReplyVariants ? material : nil
|
|
if expectsReplyVariants {
|
|
instruction += "\n\(replyVariantsOutputContract(for: replyVariantSet))"
|
|
}
|
|
if skill.kind == .export {
|
|
instruction += "\nPreserve the source language, addresses, names, and proper nouns."
|
|
}
|
|
// Recent same-chat context: only for a real reply, only when we have the
|
|
// incoming message, and only from exchanges inside the 5-minute window.
|
|
// iOS gives a keyboard no source-app identity, so time proximity is the
|
|
// approximation of "the same conversation".
|
|
let conversationContext: String? = {
|
|
guard skill.id == AIClipboardSkillCatalog.replyID,
|
|
oobeFeature == nil,
|
|
let material else { return nil }
|
|
let turns = replyFeedbackStore.recentConversationTurns(
|
|
excludingIncoming: material
|
|
)
|
|
return replyConversationContext(from: turns)
|
|
}()
|
|
if conversationContext != nil {
|
|
instruction += "\n" + Self.replyConversationContextGuide
|
|
}
|
|
AIAgentShortcutRun.trace("keyboard.submit skill=\(skill.id) kind=\(skill.kind)")
|
|
if let material {
|
|
AIAgentShortcutRun.traceBody("keyboard.clipboard", material)
|
|
} else {
|
|
AIAgentShortcutRun.trace("keyboard.clipboard missing")
|
|
}
|
|
if conversationContext != nil {
|
|
AIAgentShortcutRun.trace("keyboard.replyContext attached")
|
|
}
|
|
let resolution = AIClipboardPrompt.resolve(
|
|
instruction: instruction,
|
|
material: material,
|
|
conversationContext: conversationContext
|
|
)
|
|
if case .materialUnavailable = resolution {
|
|
AIAgentShortcutRun.trace("keyboard.submit rejected clipboardUnavailable skill=\(skill.id)")
|
|
}
|
|
if skill.id == AIClipboardSkillCatalog.summarizeWebPageID,
|
|
let material,
|
|
let url = ClipboardWebLinkResolver.singleWebURL(in: material) {
|
|
submitResolvedPrompt(
|
|
.ready(instruction),
|
|
taskKind: skill.managedGatewayTaskKind,
|
|
oobeFeature: oobeFeature,
|
|
thinkingEnabled: skill.thinkingEnabled,
|
|
webPageURL: url
|
|
)
|
|
return
|
|
}
|
|
if skill.id == AIClipboardSkillCatalog.summarizeWebPageID {
|
|
submitResolvedPrompt(
|
|
.materialUnavailable,
|
|
taskKind: skill.managedGatewayTaskKind,
|
|
oobeFeature: oobeFeature,
|
|
thinkingEnabled: skill.thinkingEnabled
|
|
)
|
|
return
|
|
}
|
|
submitResolvedPrompt(
|
|
resolution,
|
|
taskKind: skill.managedGatewayTaskKind,
|
|
oobeFeature: oobeFeature,
|
|
thinkingEnabled: skill.thinkingEnabled,
|
|
expectsReplyVariants: expectsReplyVariants
|
|
)
|
|
}
|
|
|
|
/// Tap an idle hint card: resolve its material, skip the mic, ask the host.
|
|
func submitHintCard(_ card: AIHintCard) {
|
|
guard canAcceptIdleSubmit else { return }
|
|
// The OOBE ask-AI lesson must use the explicit hold-to-talk path. Idle
|
|
// cards can otherwise pull unrelated clipboard material into a request.
|
|
guard state.oobePracticeSession == nil else { return }
|
|
prepareConversationForRequest()
|
|
let resolution = AIHintPool.resolvePrompt(
|
|
for: card,
|
|
clipboardText: ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
|
|
)
|
|
submitResolvedPrompt(
|
|
resolution,
|
|
taskKind: card.taskKind,
|
|
requestSource: .hotword,
|
|
oobeFeature: nil,
|
|
thinkingEnabled: true
|
|
)
|
|
}
|
|
|
|
private var canAcceptIdleSubmit: Bool {
|
|
switch state.aiSession.phase {
|
|
case .inactive, .idle, .awaitingSend, .inserted, .sent, .failed:
|
|
return true
|
|
case .preparing, .listening, .recognizing, .generating,
|
|
.ready:
|
|
return false
|
|
}
|
|
}
|
|
|
|
private func submitResolvedPrompt(
|
|
_ resolution: AIClipboardPrompt.Resolution,
|
|
taskKind: ManagedGatewayTaskKind,
|
|
requestSource: ManagedGatewayRequestSource? = nil,
|
|
oobeFeature: ManagedGatewayOOBEFeature? = nil,
|
|
thinkingEnabled: Bool? = nil,
|
|
webPageURL: URL? = nil,
|
|
expectsReplyVariants: Bool = false
|
|
) {
|
|
guard case .ready(let prompt) = resolution else {
|
|
// The clipboard window closed between rendering and this tap.
|
|
clearPendingExportSkill()
|
|
state.aiSession.fail(
|
|
ExtL10n.string("keyboard.ai.error.clipboardUnavailable"),
|
|
utteranceID: nil
|
|
)
|
|
return
|
|
}
|
|
guard let conversationID = state.aiSession.conversationID else {
|
|
clearPendingExportSkill()
|
|
return
|
|
}
|
|
requestOOBEFeature = oobeFeature
|
|
requestExpectsReplyVariants = expectsReplyVariants
|
|
let disposition = flow.submitAIQuestion(
|
|
text: prompt,
|
|
conversationID: conversationID,
|
|
taskKind: taskKind,
|
|
requestSource: requestSource,
|
|
oobeFeature: oobeFeature,
|
|
thinkingEnabled: thinkingEnabled,
|
|
webPageURL: webPageURL
|
|
)
|
|
if case .rejected(let rejection) = disposition {
|
|
clearPendingExportSkill()
|
|
requestExpectsReplyVariants = false
|
|
requestReplyVariantSet = .generic
|
|
requestReplySourceText = nil
|
|
requestReplyFeedbackSource = nil
|
|
state.aiSession.fail(message(for: rejection), utteranceID: nil)
|
|
}
|
|
}
|
|
|
|
func cancel() {
|
|
guard state.aiSession.isBusy else { return }
|
|
clearPendingExportSkill()
|
|
requestInsertionFingerprint = nil
|
|
requestOOBEFeature = nil
|
|
requestExpectsReplyVariants = false
|
|
requestReplyVariantSet = .generic
|
|
requestReplySourceText = nil
|
|
flow.cancelAIRecording()
|
|
state.aiSession.cancelCurrentWork()
|
|
}
|
|
|
|
func confirmPendingAnswer() {
|
|
if state.aiSession.canInsert, let answer = state.aiSession.answer {
|
|
guard insertAnswer(answer) else { return }
|
|
recordReplySelection(answer: answer)
|
|
markOOBECompletedAfterInsertion()
|
|
state.aiSession.markAnswerInserted(
|
|
offersSend: state.returnKeyRole.usesActionFill
|
|
)
|
|
conversationInsertionFingerprint = captureInsertionFingerprint()
|
|
hasConversationInsertionTarget = true
|
|
resetStructuredReplyConversationIfNeeded()
|
|
restoreAutoReplySurfaceIfNeeded()
|
|
}
|
|
}
|
|
|
|
func selectReplyVariant(id: UUID) {
|
|
guard let answer = state.aiSession.selectReplyVariant(id: id),
|
|
insertAnswer(answer) else {
|
|
return
|
|
}
|
|
recordReplySelection(answer: answer)
|
|
markOOBECompletedAfterInsertion()
|
|
state.aiSession.markAnswerInserted(
|
|
offersSend: state.returnKeyRole.usesActionFill
|
|
)
|
|
conversationInsertionFingerprint = captureInsertionFingerprint()
|
|
hasConversationInsertionTarget = true
|
|
resetStructuredReplyConversationIfNeeded()
|
|
restoreAutoReplySurfaceIfNeeded()
|
|
}
|
|
|
|
func discardPendingAnswer() {
|
|
discardPendingReplyFeedback()
|
|
state.aiSession.discardReadyAnswer()
|
|
requestInsertionFingerprint = nil
|
|
requestOOBEFeature = nil
|
|
resetStructuredReplyConversationIfNeeded()
|
|
// Tapping the result's X means the user is done with this clipboard
|
|
// generation — retire the whole assistant affordance for it. This routes
|
|
// through the shared dismiss so it both marks the skill top bar dismissed
|
|
// (`dismissedClipboardSkillEntryID`) AND clears the standalone paste
|
|
// suggestion (`clipboardSuggestionText`). Setting only the skill marker
|
|
// left the paste capsule + X alive on the surface we return to below,
|
|
// because the two are independent state.
|
|
state.dismissClipboardSuggestion()
|
|
restoreAutoReplySurfaceIfNeeded()
|
|
}
|
|
|
|
/// If auto mode moved the user onto the assistant surface to show a reply,
|
|
/// finishing it — using a reply or closing — returns them to the keyboard
|
|
/// they were last using instead of leaving them on voice input.
|
|
private func restoreAutoReplySurfaceIfNeeded() {
|
|
// A read-only auto result (auto-translate) is finished once dismissed.
|
|
state.autoResultReadOnly = false
|
|
// The result panel is closing, so its "other skills" row retires too.
|
|
state.activeClipboardSkillID = nil
|
|
guard let surface = state.autoReplyReturnSurface else { return }
|
|
state.autoReplyReturnSurface = nil
|
|
state.setSurface(surface)
|
|
}
|
|
|
|
func performCurrentFieldAction() {
|
|
guard state.assistantActionAvailable else { return }
|
|
if state.aiSession.canSend {
|
|
state.aiSession.markAnswerSent()
|
|
}
|
|
// Let the host consume the inserted answer before issuing Return.
|
|
Task { @MainActor [weak self] in
|
|
await Task.yield()
|
|
self?.performReturn()
|
|
}
|
|
}
|
|
|
|
func utterancePrepared(_ utteranceID: UUID) {
|
|
state.aiSession.beginPreparing(utteranceID: utteranceID)
|
|
}
|
|
|
|
func recordingStarted(_ utteranceID: UUID) {
|
|
state.aiSession.beginListening(utteranceID: utteranceID)
|
|
}
|
|
|
|
func recognitionStarted(_ utteranceID: UUID) {
|
|
state.aiSession.beginRecognizing(utteranceID: utteranceID)
|
|
}
|
|
|
|
func generatingStarted(_ utteranceID: UUID) {
|
|
state.aiSession.beginGenerating(question: "", utteranceID: utteranceID)
|
|
}
|
|
|
|
func receiveTranscript(
|
|
_ transcript: String,
|
|
utteranceID: UUID,
|
|
status: FlowResult.Status
|
|
) {
|
|
if AIClipboardPrompt.isInternalPrompt(transcript) {
|
|
if status == .rawReady {
|
|
state.aiSession.beginGenerating(question: "", utteranceID: utteranceID)
|
|
}
|
|
return
|
|
}
|
|
state.aiSession.updateTranscript(transcript, utteranceID: utteranceID)
|
|
if status == .rawReady {
|
|
state.aiSession.beginGenerating(
|
|
question: transcript,
|
|
utteranceID: utteranceID
|
|
)
|
|
}
|
|
}
|
|
|
|
func receivePartialAnswer(_ draft: String, utteranceID: UUID) {
|
|
if isPendingExportSkill || requestExpectsReplyVariants { return }
|
|
state.aiSession.receivePartialAnswer(draft, utteranceID: utteranceID)
|
|
}
|
|
|
|
func receive(result: FlowResult) {
|
|
guard result.resolvedUtteranceMode == .aiQuestion else {
|
|
return
|
|
}
|
|
if isPendingExportSkill {
|
|
guard result.aiConversationID == state.aiSession.conversationID else { return }
|
|
finishExportSkill(answer: result.text ?? "")
|
|
return
|
|
}
|
|
guard result.aiConversationID == state.aiSession.conversationID,
|
|
let answer = result.text,
|
|
!answer.isEmpty else {
|
|
requestExpectsReplyVariants = false
|
|
requestReplyVariantSet = .generic
|
|
requestReplySourceText = nil
|
|
requestReplyFeedbackSource = nil
|
|
state.aiSession.fail(
|
|
ExtL10n.string("keyboard.ai.error.requestFailed"),
|
|
utteranceID: result.utteranceId
|
|
)
|
|
return
|
|
}
|
|
if requestExpectsReplyVariants {
|
|
requestExpectsReplyVariants = false
|
|
requestInsertionFingerprint = nil
|
|
let sourceText = requestReplySourceText
|
|
let variantSet = requestReplyVariantSet
|
|
requestReplyVariantSet = .generic
|
|
requestReplySourceText = nil
|
|
switch AIReplyVariantParser.parseOrFallback(
|
|
answer,
|
|
sourceText: sourceText,
|
|
variantSet: variantSet
|
|
) {
|
|
case .variants(let variants):
|
|
state.aiSession.receiveReplyVariants(
|
|
variants,
|
|
utteranceID: result.utteranceId
|
|
)
|
|
beginReplyFeedback(variants: variants)
|
|
pendingStructuredReplyResult = true
|
|
case .single(let fallback):
|
|
// A malformed structured response stays explicit-review only.
|
|
// Never auto-insert model JSON or a code fence into the host.
|
|
state.aiSession.receiveAnswer(
|
|
fallback.text,
|
|
utteranceID: result.utteranceId
|
|
)
|
|
if let answer = state.aiSession.answer {
|
|
beginReplyFeedback(answer: answer)
|
|
}
|
|
pendingStructuredReplyResult = true
|
|
case nil:
|
|
requestReplyFeedbackSource = nil
|
|
state.aiSession.fail(
|
|
ExtL10n.string("keyboard.ai.error.requestFailed"),
|
|
utteranceID: result.utteranceId
|
|
)
|
|
}
|
|
return
|
|
}
|
|
state.aiSession.receiveAnswer(answer, utteranceID: result.utteranceId)
|
|
if let answer = state.aiSession.answer {
|
|
beginReplyFeedback(answer: answer)
|
|
}
|
|
defer { requestInsertionFingerprint = nil }
|
|
// Auto-translate is staged like a reply: show the result on the keyboard
|
|
// and let the user drop it into the field with the glass Insert button,
|
|
// never straight into the host field.
|
|
if state.autoResultReadOnly {
|
|
return
|
|
}
|
|
guard state.aiSession.canInsert,
|
|
let expected = requestInsertionFingerprint,
|
|
captureInsertionFingerprint() == expected,
|
|
let answer = state.aiSession.answer,
|
|
insertAnswer(answer) else {
|
|
// Keep `.ready`: the unified UI presents an explicit Insert / Discard
|
|
// fallback when the field or caret changed during generation.
|
|
return
|
|
}
|
|
recordReplySelection(answer: answer)
|
|
markOOBECompletedAfterInsertion()
|
|
state.aiSession.markAnswerInserted(
|
|
offersSend: state.returnKeyRole.usesActionFill
|
|
)
|
|
conversationInsertionFingerprint = captureInsertionFingerprint()
|
|
hasConversationInsertionTarget = true
|
|
}
|
|
|
|
func fail(_ message: String, utteranceID: UUID?) {
|
|
clearPendingExportSkill()
|
|
requestInsertionFingerprint = nil
|
|
requestOOBEFeature = nil
|
|
requestExpectsReplyVariants = false
|
|
requestReplyVariantSet = .generic
|
|
requestReplySourceText = nil
|
|
requestReplyFeedbackSource = nil
|
|
state.aiSession.fail(message, utteranceID: utteranceID)
|
|
}
|
|
|
|
/// A global output-language change starts a clean conversation so retained
|
|
/// turns cannot override the newly selected language policy.
|
|
func resetConversationForConfigurationChange() {
|
|
guard state.aiSession.isActive else { return }
|
|
beginNewPresentation()
|
|
}
|
|
|
|
private func endConversationIfNeeded() {
|
|
clearPendingExportSkill()
|
|
guard let conversationID = state.aiSession.conversationID else { return }
|
|
flow.endAIConversation(conversationID)
|
|
}
|
|
|
|
private func clearPendingExportSkill() {
|
|
state.pendingClipboardSkillID = nil
|
|
state.pendingClipboardSkillSource = nil
|
|
}
|
|
|
|
private func prepareConversationForRequest() {
|
|
// Any fresh request drops the previous result's "active skill" marker;
|
|
// submitClipboardSkill re-sets it right after calling this.
|
|
state.activeClipboardSkillID = nil
|
|
requestExpectsReplyVariants = false
|
|
requestReplyVariantSet = .generic
|
|
requestReplySourceText = nil
|
|
requestReplyFeedbackSource = nil
|
|
pendingStructuredReplyResult = false
|
|
let currentFingerprint = captureInsertionFingerprint()
|
|
if state.aiSession.isActive,
|
|
hasConversationInsertionTarget,
|
|
conversationInsertionFingerprint != currentFingerprint {
|
|
beginNewPresentation()
|
|
} else {
|
|
enterIfNeeded()
|
|
}
|
|
conversationInsertionFingerprint = currentFingerprint
|
|
hasConversationInsertionTarget = true
|
|
requestInsertionFingerprint = currentFingerprint
|
|
}
|
|
|
|
/// Tells the model how to read the untrusted `<conversation_context>` block.
|
|
/// English to match the other inline directives in this file.
|
|
private static let replyConversationContextGuide =
|
|
"A <conversation_context> block precedes the clipboard text: it is the "
|
|
+ "earlier back-and-forth in this same chat, where \"Them:\" is the "
|
|
+ "other person and \"You:\" is the user. Use it only as background to "
|
|
+ "keep the reply consistent and on-topic; reply to the latest message "
|
|
+ "in clipboard_text, and never quote or answer the context lines "
|
|
+ "themselves."
|
|
|
|
/// Formats recent turns into the plain `Them:/You:` transcript embedded in
|
|
/// the prompt. Each line is length-capped so a long history can't crowd out
|
|
/// the message being replied to. Returns nil when there is nothing to add.
|
|
private func replyConversationContext(
|
|
from turns: [ClipboardReplyFeedbackStore.ConversationTurn]
|
|
) -> String? {
|
|
guard !turns.isEmpty else { return nil }
|
|
let cap = 500
|
|
let lines = turns.flatMap { turn -> [String] in
|
|
[
|
|
"Them: " + String(turn.incoming.prefix(cap)),
|
|
"You: " + String(turn.reply.prefix(cap))
|
|
]
|
|
}
|
|
return lines.joined(separator: "\n")
|
|
}
|
|
|
|
private func beginReplyFeedback(variants: [AIReplyVariant]) {
|
|
guard let sourceText = requestReplyFeedbackSource else { return }
|
|
requestReplyFeedbackSource = nil
|
|
let snapshots = variants.map { variant in
|
|
ClipboardReplyCandidateSnapshot(
|
|
id: variant.id,
|
|
kind: feedbackKind(for: variant.kind),
|
|
text: variant.text,
|
|
emotion: variant.emotion.rawValue
|
|
)
|
|
}
|
|
pendingReplyFeedbackRecordID = replyFeedbackStore.begin(
|
|
sourceText: sourceText,
|
|
candidates: snapshots,
|
|
styleID: state.clipboardReplyStyle?.styleID
|
|
)
|
|
}
|
|
|
|
private func beginReplyFeedback(answer: AIAnswer) {
|
|
beginReplyFeedback(
|
|
variants: [
|
|
AIReplyVariant(
|
|
id: answer.id,
|
|
kind: .ordinary,
|
|
emotion: .neutral,
|
|
text: answer.text
|
|
)
|
|
]
|
|
)
|
|
}
|
|
|
|
private func recordReplySelection(answer: AIAnswer) {
|
|
guard let recordID = pendingReplyFeedbackRecordID else { return }
|
|
let candidateID = state.aiSession.selectedReplyVariant?.id ?? answer.id
|
|
replyFeedbackStore.recordSelection(
|
|
recordID: recordID,
|
|
candidateID: candidateID,
|
|
answerID: answer.id
|
|
)
|
|
pendingReplyFeedbackRecordID = nil
|
|
}
|
|
|
|
private func discardPendingReplyFeedback() {
|
|
guard let recordID = pendingReplyFeedbackRecordID else { return }
|
|
replyFeedbackStore.recordDiscard(recordID: recordID)
|
|
pendingReplyFeedbackRecordID = nil
|
|
}
|
|
|
|
private func feedbackKind(
|
|
for kind: AIReplyVariant.Kind
|
|
) -> ClipboardReplyCandidateSnapshot.Kind {
|
|
guard let snapshotKind = ClipboardReplyCandidateSnapshot.Kind(
|
|
rawValue: kind.rawValue
|
|
) else {
|
|
assertionFailure("Unmapped reply variant kind: \(kind.rawValue)")
|
|
return .ordinary
|
|
}
|
|
return snapshotKind
|
|
}
|
|
|
|
/// The host conversation contains the structured JSON result rather than
|
|
/// the chosen reply. Start a clean in-memory turn after selection/discard
|
|
/// so a later spoken follow-up cannot treat that JSON as chat history.
|
|
private func resetStructuredReplyConversationIfNeeded() {
|
|
guard pendingStructuredReplyResult,
|
|
state.aiSession.isActive else {
|
|
return
|
|
}
|
|
pendingStructuredReplyResult = false
|
|
endConversationIfNeeded()
|
|
state.aiSession.resetConversationPreservingAnswer()
|
|
}
|
|
|
|
private func replyVariantsOutputContract(
|
|
for variantSet: AIReplyVariantSet
|
|
) -> String {
|
|
let items = variantSet.kinds.map {
|
|
#"{"kind":"\#($0.rawValue)","emotion":"neutral","text":"..."}"#
|
|
}.joined(separator: ",")
|
|
let roleGuidance: String
|
|
switch variantSet {
|
|
case .generic:
|
|
roleGuidance = """
|
|
All three must keep the same semantic stance, facts, and level of commitment.
|
|
ordinary: natural for the situation; add emoji only when context makes it useful.
|
|
formal: professional and natural; add no new emoji by default.
|
|
playful: relaxed and fun. Emoji has no fixed numeric cap, may be varied when context supports it, must not become meaningless stacking, and must not default to using only 😂. This playful emoji rule overrides any personal no-emoji preference.
|
|
"""
|
|
case .invitation:
|
|
roleGuidance = """
|
|
invitationAccept: naturally accept without inventing availability or commitments.
|
|
invitationDecline: politely decline without inventing a reason.
|
|
invitationTentative: stay undecided and say only that confirmation is needed.
|
|
"""
|
|
case .task:
|
|
roleGuidance = """
|
|
taskAcknowledge: acknowledge only source-supported work and timing.
|
|
taskClarify: ask only the most important missing detail.
|
|
taskNegotiate: negotiate scope or timing without inventing constraints.
|
|
"""
|
|
case .blessing:
|
|
roleGuidance = """
|
|
blessingReturn: sincerely thank and return an appropriate wish.
|
|
blessingWarm: give a concise, warm response.
|
|
blessingPlayful: respond lightly and playfully when the context is safe.
|
|
"""
|
|
case .clarification:
|
|
roleGuidance = """
|
|
clarificationDirect: answer only the part supported by available context.
|
|
clarificationQuestion: ask one essential missing question.
|
|
clarificationConfirm: briefly confirm understanding, then ask the key question.
|
|
"""
|
|
case .yesNoQuestion:
|
|
roleGuidance = """
|
|
answerAffirmative: answer yes / it works, only when the source supports it.
|
|
answerNegative: answer no / it does not, when the source implies a problem or blocker.
|
|
answerConditional: answer that it depends, stating the specific condition.
|
|
answerDefer: say you need to confirm first, without inventing when.
|
|
"""
|
|
}
|
|
return """
|
|
MULTI-REPLY OUTPUT CONTRACT (highest priority):
|
|
Return only one valid JSON object with exactly this shape and no Markdown fence or extra keys:
|
|
{"variants":[\(items)]}
|
|
Include exactly these \(variantSet.kinds.count) kinds in the shown order. Every text must be a complete reply in the source language.
|
|
\(roleGuidance)
|
|
Every item must advance the conversation with a reaction, answer, question, decision, or next step. Never restate, paraphrase, summarize, or synonymically rewrite the clipboard text. In particular, do not begin a reply by repeating the source's subject and event. For a declarative update, react to its implication or emotion instead of reporting the update back to its sender.
|
|
Apply any <reply_scene> constraint to every item. It overrides the kind-specific tone guidance below when they conflict.
|
|
Apply the existing <user_reply_style> wording, rhythm, and stable habits to every item without changing these rules.
|
|
emotion must be exactly one of: neutral, warm, celebratory, empathetic, encouraging, grateful, apologetic, reassuring, playful, enthusiastic, calm. The app, not the model, chooses all icons.
|
|
"""
|
|
}
|
|
|
|
private func oobeFeature(for skill: AIClipboardSkill) -> ManagedGatewayOOBEFeature? {
|
|
switch skill.id {
|
|
case AIClipboardSkillCatalog.replyID:
|
|
return expectedOOBEFeature(.clipboardReply)
|
|
case AIClipboardSkillCatalog.translateID:
|
|
return expectedOOBEFeature(.clipboardTranslate)
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
private func expectedOOBEFeature(
|
|
_ feature: ManagedGatewayOOBEFeature
|
|
) -> ManagedGatewayOOBEFeature? {
|
|
guard state.oobePracticeSession?.expectedFeature == feature else { return nil }
|
|
return feature
|
|
}
|
|
|
|
private func oobeMaterial(for feature: ManagedGatewayOOBEFeature?) -> String? {
|
|
guard let feature,
|
|
feature == .clipboardReply || feature == .clipboardTranslate,
|
|
let session = state.oobePracticeSession else {
|
|
return nil
|
|
}
|
|
return KeyboardSetupBridge.oobeClipboardMaterial(sessionID: session.sessionID)
|
|
}
|
|
|
|
private func markOOBECompletedAfterInsertion() {
|
|
guard let feature = requestOOBEFeature,
|
|
let session = state.oobePracticeSession else {
|
|
return
|
|
}
|
|
_ = KeyboardSetupBridge.markOOBEPracticeCompleted(
|
|
sessionID: session.sessionID,
|
|
feature: feature
|
|
)
|
|
}
|
|
|
|
private var isPendingExportSkill: Bool {
|
|
guard let id = state.pendingClipboardSkillID else { return false }
|
|
return resolvedSkill(id: id)?.kind == .export
|
|
}
|
|
|
|
private func resolvedSkill(id: String) -> AIClipboardSkill? {
|
|
state.clipboardSkillCatalog.first { $0.id == id }
|
|
}
|
|
|
|
/// Parse an export skill. Empty → in-keyboard tip, stay in the host app.
|
|
/// Lines → hand off to the host (Shortcut, Maps, or Didi).
|
|
private func finishExportSkill(answer: String) {
|
|
let source = state.pendingClipboardSkillSource
|
|
?? ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
|
|
let skillID = state.pendingClipboardSkillID
|
|
let items: [String]
|
|
let emptyTipKey: String
|
|
switch skillID {
|
|
case AIClipboardSkillCatalog.extractEventsID:
|
|
items = AIEventExtraction.lines(from: answer, sourceClipboard: source)
|
|
emptyTipKey = "keyboard.ai.skill.noEvents"
|
|
case AIClipboardSkillCatalog.extractTodosID:
|
|
items = AITodoExtraction.items(from: answer, sourceClipboard: source)
|
|
emptyTipKey = "keyboard.ai.skill.noTodos"
|
|
case AIClipboardSkillCatalog.navigateID:
|
|
items = AIAddressExtraction.lines(from: answer, sourceClipboard: source)
|
|
emptyTipKey = "keyboard.ai.skill.noAddress"
|
|
case AIClipboardSkillCatalog.saveToNotesID:
|
|
items = AINoteExport.items(
|
|
from: answer,
|
|
sourceClipboard: source,
|
|
locale: AIHintLocaleResolver.packLocale()
|
|
)
|
|
emptyTipKey = "keyboard.ai.skill.noNote"
|
|
default:
|
|
items = AIGenericSkillExport.items(from: answer)
|
|
emptyTipKey = "keyboard.ai.skill.noExportItems"
|
|
}
|
|
AIAgentShortcutRun.traceBody("keyboard.llmRaw", answer)
|
|
AIAgentShortcutRun.trace(
|
|
"keyboard.parse items=\(items.count) clipboardChars=\(source?.count ?? 0)"
|
|
)
|
|
#if DEBUG
|
|
if !items.isEmpty {
|
|
AIAgentShortcutRun.traceBody("keyboard.parsedTitles", items.joined(separator: "\n"))
|
|
}
|
|
#endif
|
|
clearPendingExportSkill()
|
|
if state.aiSession.isBusy {
|
|
state.aiSession.cancelCurrentWork()
|
|
}
|
|
if state.aiSession.isActive {
|
|
state.aiSession.resetConversationPreservingAnswer()
|
|
}
|
|
guard !items.isEmpty else {
|
|
AIAgentShortcutRun.trace("keyboard.parse empty — skip Shortcuts")
|
|
state.skillTipText = ExtL10n.string(emptyTipKey)
|
|
return
|
|
}
|
|
guard let skillID, let skill = resolvedSkill(id: skillID) else {
|
|
AIAgentShortcutRun.trace("keyboard.parse missingSkill")
|
|
state.skillTipText = ExtL10n.string("keyboard.ai.skill.shortcutMissing")
|
|
return
|
|
}
|
|
if skill.requiresShortcut, skill.shortcutName == nil {
|
|
AIAgentShortcutRun.trace("keyboard.parse missingShortcut skill=\(skillID)")
|
|
state.skillTipText = ExtL10n.string("keyboard.ai.skill.shortcutMissing")
|
|
return
|
|
}
|
|
AIAgentShortcutRun.trace("keyboard.handoffToHost skill=\(skillID) items=\(items.count)")
|
|
let tipKey: String
|
|
switch skillID {
|
|
case AIClipboardSkillCatalog.navigateID:
|
|
tipKey = "keyboard.ai.skill.openingMaps"
|
|
case AIClipboardSkillCatalog.extractTodosID:
|
|
tipKey = "keyboard.ai.skill.savingReminders"
|
|
case AIClipboardSkillCatalog.extractEventsID:
|
|
tipKey = "keyboard.ai.skill.savingEvents"
|
|
default:
|
|
tipKey = "keyboard.ai.skill.runningShortcut"
|
|
}
|
|
state.skillTipText = ExtL10n.string(tipKey)
|
|
state.runClipboardExportSkill(skillID, items)
|
|
}
|
|
|
|
private func message(for rejection: FlowUtteranceStartRejection) -> String {
|
|
switch rejection {
|
|
case .onboardingIncomplete:
|
|
return ExtL10n.string("keyboard.hint.finishSetupInApp")
|
|
case .missingAPIKey:
|
|
return ExtL10n.string("keyboard.ai.error.missingAPIKey")
|
|
case .noFullAccess:
|
|
return ExtL10n.string("keyboard.error.fullAccessRequired")
|
|
case .appGroupUnavailable:
|
|
return ExtL10n.string("keyboard.error.appGroupCommunication")
|
|
case .hostUnavailable:
|
|
return ExtL10n.string("keyboard.flow.hostDisconnected")
|
|
case .pipelineBusy:
|
|
return ExtL10n.string("keyboard.ai.error.pipelineBusy")
|
|
}
|
|
}
|
|
}
|