rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(keyboard): unify assistant voice and AI workflows

Browse Source 
Merge dictation and AI controls into one assistant surface, preserve safe insertion and clipboard actions, and align settings and tests with the new flow.
This commit is contained in:
Rocky
2026-08-16 12:07:06 +08:00
parent 46f6d818b8
commit fb97ec2937
45 changed files with 1969 additions and 2143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OSGKeyboardExt/Services/AIKeyboardCoordinator.swift
+83 -17
View File
@@ -14,22 +14,31 @@ final class AIKeyboardCoordinator {
private let flow: KeyboardFlowCoordinator
private let insertAnswer: (AIAnswer) -> Bool
private let performReturn: () -> Void
private let captureInsertionFingerprint: () -> String?
private var requestInsertionFingerprint: String?
private var conversationInsertionFingerprint: String?
private var hasConversationInsertionTarget = false
init(
state: KeyboardState,
flow: KeyboardFlowCoordinator,
insertAnswer: @escaping (AIAnswer) -> Bool,
performReturn: @escaping () -> Void
performReturn: @escaping () -> Void,
captureInsertionFingerprint: @escaping () -> String?
) {
self.state = state
self.flow = flow
self.insertAnswer = insertAnswer
self.performReturn = performReturn
self.captureInsertionFingerprint = captureInsertionFingerprint
}
func beginNewPresentation() {
endConversationIfNeeded()
state.aiSession.enter()
requestInsertionFingerprint = nil
conversationInsertionFingerprint = nil
hasConversationInsertionTarget = false
}
func enterIfNeeded() {
@@ -43,6 +52,9 @@ final class AIKeyboardCoordinator {
}
endConversationIfNeeded()
state.aiSession.leave()
requestInsertionFingerprint = nil
conversationInsertionFingerprint = nil
hasConversationInsertionTarget = false
}
func toggleMicrophone() {
@@ -51,17 +63,17 @@ final class AIKeyboardCoordinator {
guard let utteranceID = state.aiSession.activeUtteranceID else { return }
state.aiSession.beginRecognizing(utteranceID: utteranceID)
flow.stopAIRecording()
case .idle, .ready, .awaitingSend, .inserted, .sent, .failed:
enterIfNeeded()
case .idle, .awaitingSend, .inserted, .sent, .failed:
prepareConversationForRequest()
guard let conversationID = state.aiSession.conversationID else { return }
let disposition = flow.beginAIRecording(conversationID: conversationID)
if case .rejected(let rejection) = disposition {
state.aiSession.fail(message(for: rejection), utteranceID: nil)
}
case .inactive:
enterIfNeeded()
prepareConversationForRequest()
toggleMicrophone()
case .preparing, .recognizing, .generating:
case .preparing, .recognizing, .generating, .ready:
break
}
}
@@ -69,7 +81,7 @@ final class AIKeyboardCoordinator {
/// Tap a clipboard skill chip: same fail-closed material path as hint cards.
func submitClipboardSkill(_ skill: AIClipboardSkill) {
guard canAcceptIdleSubmit else { return }
enterIfNeeded()
prepareConversationForRequest()
let material = ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
if skill.kind == .export {
state.pendingClipboardSkillID = skill.id
@@ -77,11 +89,14 @@ final class AIKeyboardCoordinator {
} else {
clearPendingExportSkill()
}
let instruction = AIClipboardSkillCatalog.instruction(
var instruction = AIClipboardSkillCatalog.instruction(
for: skill,
locale: AIHintLocaleResolver.packLocale(),
translationTargetLocaleId: state.translationTargetLocaleId
)
if skill.kind == .export {
instruction += "\nPreserve the source language, addresses, names, and proper nouns."
}
AIAgentShortcutRun.trace("keyboard.submit skill=\(skill.id) kind=\(skill.kind)")
if let material {
AIAgentShortcutRun.traceBody("keyboard.clipboard", material)
@@ -101,7 +116,7 @@ final class AIKeyboardCoordinator {
/// Tap an idle hint card: resolve its material, skip the mic, ask the host.
func submitHintCard(_ card: AIHintCard) {
guard canAcceptIdleSubmit else { return }
enterIfNeeded()
prepareConversationForRequest()
let resolution = AIHintPool.resolvePrompt(
for: card,
clipboardText: ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
@@ -111,10 +126,10 @@ final class AIKeyboardCoordinator {
private var canAcceptIdleSubmit: Bool {
switch state.aiSession.phase {
case .inactive, .idle, .failed:
case .inactive, .idle, .awaitingSend, .inserted, .sent, .failed:
return true
case .preparing, .listening, .recognizing, .generating,
.ready, .awaitingSend, .inserted, .sent:
.ready:
return false
}
}
@@ -150,23 +165,37 @@ final class AIKeyboardCoordinator {
func cancel() {
guard state.aiSession.isBusy else { return }
clearPendingExportSkill()
requestInsertionFingerprint = nil
flow.cancelAIRecording()
state.aiSession.cancelCurrentWork()
}
func sendLatestAnswer() {
func confirmPendingAnswer() {
if state.aiSession.canInsert, let answer = state.aiSession.answer {
guard insertAnswer(answer) else { return }
state.aiSession.markAnswerInserted(
offersSend: state.returnKeyRole.usesActionFill
)
} else if state.aiSession.canSend {
conversationInsertionFingerprint = captureInsertionFingerprint()
hasConversationInsertionTarget = true
}
}
func discardPendingAnswer() {
state.aiSession.discardReadyAnswer()
requestInsertionFingerprint = nil
}
func sendCurrentFieldAction() {
guard state.assistantSendAvailable else { return }
state.assistantSendAvailable = false
if state.aiSession.canSend {
state.aiSession.markAnswerSent()
// Let the host consume the inserted answer before issuing Return.
Task { @MainActor [weak self] in
await Task.yield()
self?.performReturn()
}
}
// Let the host consume the inserted answer before issuing Return.
Task { @MainActor [weak self] in
await Task.yield()
self?.performReturn()
}
}
@@ -230,13 +259,36 @@ final class AIKeyboardCoordinator {
return
}
state.aiSession.receiveAnswer(answer, utteranceID: result.utteranceId)
defer { requestInsertionFingerprint = nil }
guard state.aiSession.canInsert,
let expected = requestInsertionFingerprint,
captureInsertionFingerprint() == expected,
let answer = state.aiSession.answer,
insertAnswer(answer) else {
// Keep `.ready`: the unified UI presents an explicit Insert / Discard
// fallback when the field or caret changed during generation.
return
}
state.aiSession.markAnswerInserted(
offersSend: state.returnKeyRole.usesActionFill
)
conversationInsertionFingerprint = captureInsertionFingerprint()
hasConversationInsertionTarget = true
}
func fail(_ message: String, utteranceID: UUID?) {
clearPendingExportSkill()
requestInsertionFingerprint = nil
state.aiSession.fail(message, utteranceID: utteranceID)
}
/// A global output-language change starts a clean conversation so retained
/// turns cannot override the newly selected language policy.
func resetConversationForConfigurationChange() {
guard state.aiSession.isActive else { return }
beginNewPresentation()
}
private func endConversationIfNeeded() {
clearPendingExportSkill()
guard let conversationID = state.aiSession.conversationID else { return }
@@ -248,6 +300,20 @@ final class AIKeyboardCoordinator {
state.pendingClipboardSkillSource = nil
}
private func prepareConversationForRequest() {
let currentFingerprint = captureInsertionFingerprint()
if state.aiSession.isActive,
hasConversationInsertionTarget,
conversationInsertionFingerprint != currentFingerprint {
beginNewPresentation()
} else {
enterIfNeeded()
}
conversationInsertionFingerprint = currentFingerprint
hasConversationInsertionTarget = true
requestInsertionFingerprint = currentFingerprint
}
private var isPendingExportSkill: Bool {
guard let id = state.pendingClipboardSkillID else { return false }
return resolvedSkill(id: id)?.kind == .export
OSGKeyboardExt/Services/AppGroupPersistor.swift
-2
View File
@@ -40,7 +40,6 @@ public struct AppGroupPersistor {
// the keyboard stays open.
state.translationTargetLocaleId = store.translationTargetLocaleId
state.handednessPreference = store.handednessPreference
state.cursorDragNavigationEnabled = store.cursorDragNavigationEnabled
state.clipboardHistoryEnabled = store.clipboardHistoryEnabled
state.clipboardCandidateBarEnabled = store.clipboardCandidateBarEnabled
state.enabledClipboardSkillIDs = store.agentSkillLayout.enabledIDs
@@ -96,7 +95,6 @@ public struct AppGroupPersistor {
state.translationTargetLocaleId = store.translationTargetLocaleId
}
state.handednessPreference = store.handednessPreference
state.cursorDragNavigationEnabled = store.cursorDragNavigationEnabled
state.clipboardHistoryEnabled = store.clipboardHistoryEnabled
state.clipboardCandidateBarEnabled = store.clipboardCandidateBarEnabled
state.enabledClipboardSkillIDs = store.agentSkillLayout.enabledIDs
OSGKeyboardExt/Services/CursorDragController.swift
-127
View File
@@ -1,127 +0,0 @@
// CursorDragController.swift
// OSGKeyboard · Keyboard Extension
//
// Cursor-drag hint chrome and batched caret moves via textDocumentProxy.
import UIKit
import OSGKeyboardShared
@MainActor
final class CursorDragController {
private let state: KeyboardState
private let adjustTextPosition: (Int) -> Void
private weak var parentView: UIView?
private var cursorDragHintLabel: UILabel?
private var pendingHorizontalCursorSteps = 0
private var pendingVerticalCursorSteps = 0
private var cursorMoveFlushScheduled = false
private let cursorLineHaptic = UIImpactFeedbackGenerator(style: .light)
private static let cursorVerticalChunkSize = 20
init(
state: KeyboardState,
adjustTextPosition: @escaping (Int) -> Void
) {
self.state = state
self.adjustTextPosition = adjustTextPosition
}
func install(on view: UIView) {
parentView = view
let hint = UILabel()
hint.text = ExtL10n.string("keyboard.cursorDrag.centerHint")
hint.font = .systemFont(ofSize: 22, weight: .medium)
hint.textColor = UIColor.label.withAlphaComponent(0.10)
hint.textAlignment = .center
hint.numberOfLines = 1
hint.adjustsFontSizeToFitWidth = true
hint.minimumScaleFactor = 0.7
hint.isUserInteractionEnabled = false
hint.isHidden = true
hint.alpha = 0
view.addSubview(hint)
cursorDragHintLabel = hint
layoutChrome()
}
func layoutChrome() {
guard let view = parentView else { return }
cursorDragHintLabel?.frame = view.bounds
}
func setCursorDragActive(_ active: Bool) {
state.cursorDragActive = active
updateCursorDragWash(active: active)
}
func moveCursorHorizontally(by steps: Int) {
guard steps != 0 else { return }
pendingHorizontalCursorSteps += steps
scheduleCursorMoveFlush()
}
func moveCursorVertically(by steps: Int) {
guard steps != 0 else { return }
pendingVerticalCursorSteps += steps
scheduleCursorMoveFlush()
}
private func updateCursorDragWash(active: Bool) {
if active {
cursorLineHaptic.prepare()
}
layoutChrome()
guard let hint = cursorDragHintLabel else { return }
if active {
hint.isHidden = false
UIView.animate(withDuration: 0.12) { hint.alpha = 1 }
} else {
UIView.animate(withDuration: 0.12, animations: { hint.alpha = 0 }) { [weak self] _ in
guard let self, !self.state.cursorDragActive else { return }
hint.isHidden = true
}
}
}
private func scheduleCursorMoveFlush() {
guard !cursorMoveFlushScheduled else { return }
cursorMoveFlushScheduled = true
DispatchQueue.main.asyncAfter(deadline: .now() + 0.012) { [weak self] in
guard let self else { return }
self.cursorMoveFlushScheduled = false
let horizontal = self.pendingHorizontalCursorSteps
let vertical = self.pendingVerticalCursorSteps
self.pendingHorizontalCursorSteps = 0
self.pendingVerticalCursorSteps = 0
if horizontal != 0 {
OSGLog.keyboardExt.info("adjustTextPosition h=\(horizontal)")
self.adjustTextPosition(horizontal)
}
if vertical != 0 {
self.applyVerticalCursorSteps(vertical)
}
if self.pendingHorizontalCursorSteps != 0 || self.pendingVerticalCursorSteps != 0 {
self.scheduleCursorMoveFlush()
}
}
}
private func applyVerticalCursorSteps(_ steps: Int) {
let direction = steps > 0 ? 1 : -1
var remaining = abs(steps)
let chunk = Self.cursorVerticalChunkSize
while remaining > 0 {
adjustTextPosition(direction * chunk)
cursorLineHaptic.impactOccurred()
cursorLineHaptic.prepare()
remaining -= 1
}
}
}
OSGKeyboardExt/Services/KeyboardConfigSync.swift
+1 -3
View File
@@ -64,9 +64,7 @@ final class KeyboardConfigSync {
func loadPersistedConfig() -> AppGroupLoadResult {
switch persistor.load(into: state) {
case .loaded:
OSGLog.keyboardExt.info(
"config loaded — cursorDragNavigationEnabled=\(self.state.cursorDragNavigationEnabled)"
)
OSGLog.keyboardExt.info("config loaded")
syncOnboardingStateFromAppGroup()
return .loaded
case .unavailable:
OSGKeyboardExt/Services/KeyboardTextInserter.swift
+34 -4
View File
@@ -37,6 +37,7 @@ final class KeyboardTextInserter {
/// Suppresses availability refresh while we walk `deleteBackward`
/// for undo, so intermediate contexts don't flicker the button.
private var isUndoing = false
private var successPulseTask: Task<Void, Never>?
init(
state: KeyboardState,
@@ -155,6 +156,7 @@ final class KeyboardTextInserter {
// The paste pushed any previous input away from the caret, so the
// "editable last input" hint no longer applies.
clearEditHintIfPositive()
state.editAvailable = false
OSGLog.keyboardExt.info("clipboard insert length=\(text.count, privacy: .public)")
}
@@ -184,6 +186,9 @@ final class KeyboardTextInserter {
redoContextBefore = contextBeforeInput()
lastInsertedText = nil
state.undoAvailable = false
state.editAvailable = false
state.assistantSendAvailable = false
EditableInputReferenceStore.clear()
OSGLog.keyboardExt.info("undo length=\(text.count, privacy: .public)")
}
@@ -262,6 +267,10 @@ final class KeyboardTextInserter {
if state.cutAvailable != hasSelection {
state.cutAvailable = hasSelection
}
let editAvailable = editableReference() != nil
if state.editAvailable != editAvailable {
state.editAvailable = editAvailable
}
}
func editableReference() -> EditableInputReference? {
@@ -288,15 +297,25 @@ final class KeyboardTextInserter {
preceding.hasSuffix(reference.insertedText) else {
return nil
}
// The current extension instance owns an exact in-memory insertion
// record. Prefer it before the cross-process field fingerprint:
// UITextDocumentProxy can publish its updated surrounding context one
// callback after `insertText`, which otherwise makes a freshly shown
// Edit button disappear or reject its first tap.
if reference.matchesLiveInsertion(
extensionInstanceID: extensionInstanceID,
lastInsertedText: lastInsertedText,
contextBeforeInput: preceding
) {
return reference
}
guard reference.postInsertionFingerprint == nil
|| reference.postInsertionFingerprint
== fieldContextProvider()?.deliveryFingerprint else {
return nil
}
if reference.extensionInstanceID == extensionInstanceID,
lastInsertedText == reference.insertedText {
return reference
}
// Rebuilt extension instances have no trusted in-memory insertion
// record, so they continue to require the complete field fingerprint.
return reference.isFullyVerified(
contextBeforeInput: preceding,
fieldFingerprint: fieldContextProvider()?.deliveryFingerprint
@@ -512,6 +531,15 @@ final class KeyboardTextInserter {
extensionInstanceID: extensionInstanceID
)
)
state.editAvailable = true
state.assistantSendAvailable = state.returnKeyRole.usesActionFill
state.assistantInsertionSucceeded = true
successPulseTask?.cancel()
successPulseTask = Task { @MainActor [weak state] in
try? await Task.sleep(for: .milliseconds(450))
guard !Task.isCancelled else { return }
state?.assistantInsertionSucceeded = false
}
let hint = ExtL10n.string("keyboard.edit.hint.available")
editHintScheduler.show(
message: hint,
@@ -527,6 +555,8 @@ final class KeyboardTextInserter {
private func clearLastInsertion() {
lastInsertedText = nil
state.undoAvailable = false
state.editAvailable = false
state.assistantSendAvailable = false
EditableInputReferenceStore.clear()
}
}