feat(keyboard): add clipboard AI skills, hint keywords, and voice session fixes

Idle chips show entities with category icons; a fresh copy surfaces Reply/Summarize/Translate; abort/cancel/empty-tap no longer leave the mic stuck.
2026-08-13 15:47:32 +08:00
parent 9f308fadd2
commit 631617ee53
48 changed files with 1627 additions and 345 deletions
@@ -66,20 +66,44 @@ final class AIKeyboardCoordinator {
        }
    }

+    /// Tap a clipboard skill chip: same fail-closed material path as hint cards.
+    func submitClipboardSkill(_ skill: AIClipboardSkill) {
+        guard canAcceptIdleSubmit else { return }
+        enterIfNeeded()
+        let instruction = AIClipboardSkillCatalog.instruction(
+            for: skill,
+            locale: AIHintLocaleResolver.packLocale(),
+            translationTargetLocaleId: state.translationTargetLocaleId
+        )
+        let resolution = AIClipboardPrompt.resolve(
+            instruction: instruction,
+            material: ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
+        )
+        submitResolvedPrompt(resolution)
+    }
+
    /// Tap an idle hint card: resolve its material, skip the mic, ask the host.
    func submitHintCard(_ card: AIHintCard) {
-        switch state.aiSession.phase {
-        case .inactive, .idle, .failed:
-            break
-        case .preparing, .listening, .recognizing, .generating,
-             .ready, .awaitingSend, .inserted, .sent:
-            return
-        }
+        guard canAcceptIdleSubmit else { return }
        enterIfNeeded()
        let resolution = AIHintPool.resolvePrompt(
            for: card,
            clipboardText: ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
        )
+        submitResolvedPrompt(resolution)
+    }
+
+    private var canAcceptIdleSubmit: Bool {
+        switch state.aiSession.phase {
+        case .inactive, .idle, .failed:
+            return true
+        case .preparing, .listening, .recognizing, .generating,
+             .ready, .awaitingSend, .inserted, .sent:
+            return false
+        }
+    }
+
+    private func submitResolvedPrompt(_ resolution: AIClipboardPrompt.Resolution) {
        guard case .ready(let prompt) = resolution else {
            // The clipboard window closed between rendering and this tap.
            state.aiSession.fail(
@@ -132,11 +156,21 @@ final class AIKeyboardCoordinator {
        state.aiSession.beginRecognizing(utteranceID: utteranceID)
    }

+    func generatingStarted(_ utteranceID: UUID) {
+        state.aiSession.beginGenerating(question: "", utteranceID: utteranceID)
+    }
+
    func receiveTranscript(
        _ transcript: String,
        utteranceID: UUID,
        status: FlowResult.Status
    ) {
+        if AIClipboardPrompt.isInternalPrompt(transcript) {
+            if status == .rawReady {
+                state.aiSession.beginGenerating(question: "", utteranceID: utteranceID)
+            }
+            return
+        }
        state.aiSession.updateTranscript(transcript, utteranceID: utteranceID)
        if status == .rawReady {
            state.aiSession.beginGenerating(
@@ -61,10 +61,12 @@ final class KeyboardFlowCoordinator {
    var onAIUtterancePrepared: (UUID) -> Void = { _ in }
    var onAIRecordingStarted: (UUID) -> Void = { _ in }
    var onAIRecognitionStarted: (UUID) -> Void = { _ in }
+    var onAIGeneratingStarted: (UUID) -> Void = { _ in }
    var onAITranscript: (String, UUID, FlowResult.Status) -> Void = { _, _, _ in }
    var onAIStreamingAnswer: (String, UUID) -> Void = { _, _ in }
    var onAIResult: (FlowResult) -> Void = { _ in }
    var onAIFailure: (String, UUID?) -> Void = { _, _ in }
+    var onAICancelled: () -> Void = {}
    /// Utterance whose final result we already inserted (or failed). Prevents
    /// `adoptHostBusyStateIfNeeded` from re-entering `.processing` after a
    /// stale App Group snapshot still says `reason=processing`.
@@ -414,6 +416,29 @@ final class KeyboardFlowCoordinator {
            isFlowRecording = false
            stopUtteranceCountdown()
            ExtensionScreenWakeLock.release()
+            if FlowKeyboardAdoptBusyPolicy.isStaleDeliveredProcessing(
+                busyUtteranceId: busyId,
+                latestResult: FlowSessionBridge.latestResult(),
+                latestAck: FlowSessionBridge.latestAck()
+            ) {
+                // Already acked, result gone, host forgot to drop the gate.
+                // Abort unsticks the host. Do not await — claimTerminal already
+                // ran, so abort will not write a new result. Remember the id
+                // so this same refresh cannot re-adopt before the host poll.
+                writeCommand(.abort)
+                lastConsumedUtteranceId = busyId
+                lastStoppedUtteranceId = busyId
+                isAwaitingFlowResult = false
+                currentUtteranceId = nil
+                state.phase = .idle
+                state.lastTranscript = ""
+                traceState(
+                    "adoptHostBusy.staleProcessingReleased",
+                    extra: "utterance=\(busyId.uuidString.prefix(8))"
+                )
+                return
+            }
+            // Missing result with no matching ack is live ASR/LLM — wait.
            state.phase = .processing
            if state.lastTranscript.isEmpty {
                state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
@@ -557,6 +582,12 @@ final class KeyboardFlowCoordinator {
        case .requestingPermissions:
            break
        case .idle, .denied, .error:
+            // A second tap while still waiting for the host must cancel the
+            // pending start instead of stacking another utterance.
+            if currentUtteranceRequest != nil {
+                cancelCurrentDictation()
+                return
+            }
            _ = startUtterance(.dictation)
        case .processing:
            break
@@ -655,34 +686,8 @@ final class KeyboardFlowCoordinator {

    func cancelAIRecording() {
        guard currentUtteranceRequest?.isAIQuestion == true else { return }
-        recordWhenHostReady = false
-        recordAfterHandoff = false
-        isPendingFlowStart = false
-        flowStartDeadline = 0
-        coldStartDebouncer.reset()
-        stopHostReadyWait()
-        stopFlowWatchdog()
-        stopUtteranceCountdown()
-        ExtensionScreenWakeLock.release()
-        state.level = 0
-        state.phase = .idle
-        state.lastTranscript = ""
-
-        guard let utteranceID = currentUtteranceId,
-              isFlowRecording || isAwaitingFlowResult else {
-            clearUnissuedUtterance()
-            isFlowRecording = false
-            isAwaitingFlowResult = false
-            recomputeMicVoiceAvailability()
-            return
-        }
-
-        cancelledAIUtteranceIDs.insert(utteranceID)
-        writeCommand(.abort)
-        isFlowRecording = false
-        isAwaitingFlowResult = true
-        startFlowResultWatchdog()
-        recomputeMicVoiceAvailability()
+        prepareLocalCancel()
+        beginAwaitingAbort(tracking: &cancelledAIUtteranceIDs)
    }

    func endAIConversation(_ conversationID: UUID) {
@@ -708,11 +713,27 @@ final class KeyboardFlowCoordinator {
    }

    func cancelCurrentDictation() {
-        guard currentUtteranceRequest?.isEdit != true,
-              currentUtteranceRequest?.isAIQuestion != true else {
+        if currentUtteranceRequest?.isAIQuestion == true {
+            cancelAIRecording()
            return
        }
+        guard currentUtteranceRequest?.isEdit != true else { return }

+        prepareLocalCancel()
+        let hadIssuedTransport = currentUtteranceId != nil
+            && (isFlowRecording || isAwaitingFlowResult)
+        beginAwaitingAbort(tracking: &cancelledDictationUtteranceIDs)
+        traceState(
+            "dictation.cancelled",
+            extra: hadIssuedTransport
+                ? "utterance=\(currentUtteranceId?.uuidString.prefix(8) ?? "none")"
+                : "transport=localOnly"
+        )
+    }
+
+    /// Stop local wait/prime timers before an abort. Does not change phase;
+    /// `beginAwaitingAbort` keeps `.processing` until the host acks.
+    private func prepareLocalCancel() {
        recordWhenHostReady = false
        recordAfterHandoff = false
        isPendingFlowStart = false
@@ -723,29 +744,37 @@ final class KeyboardFlowCoordinator {
        stopUtteranceCountdown()
        ExtensionScreenWakeLock.release()
        state.level = 0
-        state.phase = .idle
        state.lastTranscript = ""
+    }

+    /// Keep the cancel chrome (X + white mic) until the host finishes abort.
+    private func beginAwaitingAbort(tracking cancelledIDs: inout Set<UUID>) {
        guard let utteranceID = currentUtteranceId,
              isFlowRecording || isAwaitingFlowResult else {
-            clearUnissuedUtterance()
-            isFlowRecording = false
-            isAwaitingFlowResult = false
-            recomputeMicVoiceAvailability()
-            traceState("dictation.cancelled", extra: "transport=localOnly")
+            finishLocalCancel()
            return
        }
-
-        cancelledDictationUtteranceIDs.insert(utteranceID)
+        if cancelledIDs.contains(utteranceID), isAwaitingFlowResult {
+            state.phase = .processing
+            recomputeMicVoiceAvailability()
+            return
+        }
+        cancelledIDs.insert(utteranceID)
        writeCommand(.abort)
        isFlowRecording = false
        isAwaitingFlowResult = true
+        state.phase = .processing
        startFlowResultWatchdog()
        recomputeMicVoiceAvailability()
-        traceState(
-            "dictation.cancelled",
-            extra: "utterance=\(utteranceID.uuidString.prefix(8))"
-        )
+    }
+
+    private func finishLocalCancel() {
+        clearUnissuedUtterance()
+        isFlowRecording = false
+        isAwaitingFlowResult = false
+        state.phase = .idle
+        state.lastTranscript = ""
+        recomputeMicVoiceAvailability()
    }

    func abortEditRecording() {
@@ -1085,9 +1114,11 @@ final class KeyboardFlowCoordinator {
            if isFlowRecording {
                writeCommand(.abort)
                ExtensionScreenWakeLock.release()
+                // Remember the aborted id so the next keyboard open cannot
+                // re-adopt a lagging host snapshot as 「识别中」.
+                lastStoppedUtteranceId = currentUtteranceId
            }
            currentUtteranceId = nil
-            lastStoppedUtteranceId = nil
            isFlowRecording = false
            isPendingFlowStart = false
            recordAfterHandoff = false
@@ -1174,6 +1205,10 @@ final class KeyboardFlowCoordinator {
               consumeCancelledDictationResultIfNeeded(result) {
                return
            }
+            if let result = matchingResult(),
+               consumeDiscardedEmptyResultIfNeeded(result) {
+                return
+            }
            if let result = matchingResult(),
               consumeCancelledEditResultIfNeeded(result) {
                return
@@ -1311,6 +1346,42 @@ final class KeyboardFlowCoordinator {
        return true
    }

+    private func consumeDiscardedEmptyResultIfNeeded(_ result: FlowResult) -> Bool {
+        guard result.errorKind == .discardedEmpty,
+              result.status == .aborted || isTerminalFailure(result) else {
+            return false
+        }
+        FlowSessionBridge.writeAck(
+            FlowAck(
+                sessionId: result.sessionId,
+                utteranceId: result.utteranceId,
+                commandSeq: result.commandSeq,
+                hostGeneration: result.hostGeneration,
+                revision: result.revision,
+                deliveryOutcome: .rejected
+            )
+        )
+        lastConsumedUtteranceId = result.utteranceId
+        lastStoppedUtteranceId = nil
+        stopUtteranceCountdown()
+        stopFlowWatchdog()
+        ExtensionScreenWakeLock.release()
+        let wasAI = currentUtteranceRequest?.isAIQuestion == true
+        resetEditTransportState()
+        state.level = 0
+        state.phase = .idle
+        state.lastTranscript = ""
+        recomputeMicVoiceAvailability()
+        if wasAI {
+            onAICancelled()
+        }
+        traceState(
+            "utterance.discardedEmptyTap",
+            extra: "utterance=\(result.utteranceId.uuidString.prefix(8))"
+        )
+        return true
+    }
+
    private func consumeCancelledEditResultIfNeeded(_ result: FlowResult) -> Bool {
        guard cancelledEditUtteranceIDs.contains(result.utteranceId),
              result.status == .final || isTerminalFailure(result) else {
@@ -1754,10 +1825,10 @@ final class KeyboardFlowCoordinator {
            writeSubmitAIQuestion(question)
            isFlowRecording = false
            isAwaitingFlowResult = true
-            state.lastTranscript = question
+            state.lastTranscript = ""
            state.phase = .processing
            if let currentUtteranceId {
-                onAIRecognitionStarted(currentUtteranceId)
+                onAIGeneratingStarted(currentUtteranceId)
            }
            startFlowResultWatchdog()
            recomputeMicVoiceAvailability()
@@ -1935,6 +2006,11 @@ final class KeyboardFlowCoordinator {
            if (result.status == .partial || result.status == .rawReady),
               let partial = result.text,
               !partial.isEmpty {
+                if result.resolvedUtteranceMode == .aiQuestion,
+                   AIClipboardPrompt.isInternalPrompt(partial) {
+                    onAIGeneratingStarted(result.utteranceId)
+                    return
+                }
                state.lastTranscript = partial
                if result.resolvedUtteranceMode == .aiQuestion {
                    onAITranscript(partial, result.utteranceId, result.status)
@@ -1982,6 +2058,10 @@ final class KeyboardFlowCoordinator {
                   self.consumeCancelledDictationResultIfNeeded(result) {
                    return
                }
+                if let result = self.matchingResult(),
+                   self.consumeDiscardedEmptyResultIfNeeded(result) {
+                    return
+                }
                if let result = self.matchingResult(),
                   self.consumeCancelledEditResultIfNeeded(result) {
                    return