rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(keyboard): unify assistant voice and AI workflows

Browse Source 
Merge dictation and AI controls into one assistant surface, preserve safe insertion and clipboard actions, and align settings and tests with the new flow.
This commit is contained in:
Rocky
2026-08-16 12:07:06 +08:00
parent 46f6d818b8
commit fb97ec2937
45 changed files with 1969 additions and 2143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OSGKeyboardShared/DesignSystem/Theme.swift
+5
View File
@@ -25,6 +25,8 @@ public struct ThemePalette: Sendable, Equatable {
public let accent: Color
public let accentMuted: Color
public let accentGlow: Color
/// Distinguishes AI listening / generation from ordinary dictation.
public let aiTeal: Color
public let danger: Color
public let success: Color
@@ -54,6 +56,7 @@ public enum Palette {
public static let accent = Color(red: 0.227, green: 0.627, blue: 0.353) // #3AA05A
public static let accentMuted = accent.opacity(0.18)
public static let accentGlow = accent.opacity(0.42)
public static let aiTeal = Color(red: 0.169, green: 0.686, blue: 0.643) // #2BAFA4
// Semantic
public static let danger = Color(red: 1.000, green: 0.271, blue: 0.227) // #FF453A
@@ -86,6 +89,7 @@ public enum Palette {
accent: accent,
accentMuted: accentMuted,
accentGlow: accentGlow,
aiTeal: aiTeal,
danger: danger,
success: success,
warning: warning,
@@ -108,6 +112,7 @@ public enum Palette {
accent: Color(red: 0.227, green: 0.627, blue: 0.353), // #3AA05A
accentMuted: Color(red: 0.227, green: 0.627, blue: 0.353).opacity(0.14),
accentGlow: Color(red: 0.227, green: 0.627, blue: 0.353).opacity(0.32),
aiTeal: Color(red: 0.169, green: 0.686, blue: 0.643), // #2BAFA4
danger: Color(red: 1.000, green: 0.231, blue: 0.188), // #FF3B30
success: Color(red: 0.227, green: 0.627, blue: 0.353), // same as accent
warning: Color(red: 1.000, green: 0.620, blue: 0.094), // #FF9E18
OSGKeyboardShared/Models/AISessionState.swift
+12
View File
@@ -190,6 +190,18 @@ public struct AISessionState: Equatable, Sendable {
phase = .sent
}
/// Drops a result retained because the insertion target changed.
/// The conversation remains active so the user can immediately ask again.
public mutating func discardReadyAnswer() {
guard phase == .ready else { return }
answer = nil
activeUtteranceID = nil
draftAnswerText = nil
transcript = ""
errorMessage = nil
phase = .idle
}
public mutating func cancelCurrentWork() {
guard isBusy else { return }
activeUtteranceID = nil
OSGKeyboardShared/Models/EditableInputReference.swift
+16
View File
@@ -60,6 +60,22 @@ public struct EditableInputReference: Codable, Equatable, Sendable {
now >= expiresAt
}
/// Same-process verification uses the exact insertion kept in memory.
/// This path intentionally does not depend on the field fingerprint,
/// because UITextDocumentProxy may publish that context one callback after
/// the insertion while the in-memory record is already authoritative.
public func matchesLiveInsertion(
extensionInstanceID currentInstanceID: UUID,
lastInsertedText: String?,
contextBeforeInput: String?
) -> Bool {
!isExpired()
&& isWithinLengthBudget
&& extensionInstanceID == currentInstanceID
&& lastInsertedText == insertedText
&& contextBeforeInput?.hasSuffix(insertedText) == true
}
/// Rebuilt extensions must match the complete inserted string at the caret
/// and, when captured, the same field fingerprint; a suffix sample is insufficient.
public func isFullyVerified(
OSGKeyboardShared/Models/KeyboardChromeLayout.swift
+2
View File
@@ -8,6 +8,8 @@ import CoreGraphics
public enum KeyboardChromeLayout {
public static let totalHeight: CGFloat = 281
public static let actionKeyHeight: CGFloat = 50
/// Shared capsule action height for unified Send and edit-mode controls.
public static let assistantActionCapsuleHeight: CGFloat = actionKeyHeight
public static let actionKeyCornerRadius: CGFloat = 10
/// Shared spacing for every bottom action row. iPad uses a custom globe
/// slot; iPhone relies on the system-provided switch below the keyboard.
OSGKeyboardShared/Models/TypingInputConfiguration.swift
+5 -4
View File
@@ -216,15 +216,16 @@ public final class TypingInputConfiguration: ObservableObject {
let store = defaults ?? AppGroup.defaultsIfAvailable
guard let store else { return (.voice, nil) }
// AI is an explicit product surface. Restore it as an empty temporary
// conversation even when the general "remember surface" toggle is off.
// Older builds persisted a dedicated AI surface. It now migrates to
// the unified assistant surface.
if store.string(forKey: Key.lastSurface) == KeyboardState.Surface.ai.rawValue {
return (.ai, nil)
return (.voice, nil)
}
if store.bool(forKey: Key.rememberLastSurface),
let raw = store.string(forKey: Key.lastSurface),
let surface = KeyboardState.Surface(rawValue: raw) {
let persistedSurface = KeyboardState.Surface(rawValue: raw) {
let surface: KeyboardState.Surface = persistedSurface == .ai ? .voice : persistedSurface
let language: TypingInputLanguage? = surface == .typing
? persistedTypingLanguage(defaults: store) ?? .chinese
: nil
OSGKeyboardShared/Services/AIQuestionService.swift
+5 -1
View File
@@ -77,7 +77,11 @@ public enum AIQuestionPromptComposer {
languageInstruction = "Reply in the language used by the user's latest question."
} else {
let language = TranslationLanguageCatalog.resolve(targetLocaleID)
languageInstruction = "Reply in \(language.promptLanguageName)."
languageInstruction = """
Reply in \(language.promptLanguageName), unless the user's latest
request explicitly asks for another output language or asks to
preserve the source language.
"""
}
return """
OSGKeyboardShared/Services/CursorNavigation.swift
-328
View File
@@ -1,328 +0,0 @@
// CursorNavigation.swift
// OSGKeyboard · Shared
//
// Pure helpers for moving the text caret from the keyboard extension.
// Horizontal moves are character-accurate. Vertical moves jump between
// *visual* lines — hard `\n` breaks and soft wraps.
//
// First-principles note: a keyboard extension only sees a bounded text
// window (`documentContext{Before,After}Input`) and can only actuate via
// `adjustTextPosition(byCharacterOffset:)`. It has NO access to the host
// field's font, width, or caret rect, so soft-wrap positions are
// fundamentally unknowable and must be *estimated*. We reduce the visible
// error two ways: (1) exact handling of hard `\n`; (2) an injectable
// per-character width so the extension can feed real font metrics (killing
// the i-vs-W column drift that a fixed 1/2 table causes). The wrap width
// itself stays a calibrated estimate.
import Foundation
import CoreGraphics
public enum CursorNavigation {
/// Advance width of a single character, in an arbitrary but consistent
/// unit (points when backed by real font metrics; abstract "units" for
/// the built-in default). Must be paired with a `lineWidth` in the same
/// unit.
public typealias CharacterWidth = @Sendable (Character) -> CGFloat
// MARK: - Layout config
/// Describes how text wraps into visual lines. `lineWidth` and the values
/// returned by `widthOf` must share the same unit.
public struct VisualLineLayoutConfig: Sendable {
/// Wrap threshold: max total width of one visual line.
public let lineWidth: CGFloat
/// Per-character advance width provider.
public let widthOf: CharacterWidth
public init(
lineWidth: CGFloat,
widthOf: @escaping CharacterWidth = CursorNavigation.defaultDisplayWidth
) {
self.lineWidth = max(1, lineWidth)
self.widthOf = widthOf
}
/// Conservative default when no field width is known.
public static let fallback = VisualLineLayoutConfig(lineWidth: 44)
}
// MARK: - Public API
/// Legacy logical column (chars since last `\n`). Kept for tests.
public static func column(before: String?) -> Int {
guard let before, !before.isEmpty else { return 0 }
if let lastNewline = before.lastIndex(of: "\n") {
return before.distance(from: before.index(after: lastNewline), to: before.endIndex)
}
return before.count
}
/// Display-column offset (in `widthOf` units) on the current visual line.
public static func visualDisplayColumn(
before: String?,
after: String?,
config: VisualLineLayoutConfig
) -> CGFloat {
let text = mergedContext(before: before, after: after)
let cursor = before?.count ?? 0
let layout = VisualLineLayout(text: text, config: config)
let lineStart = layout.lineStart(containing: cursor)
return layout.width(from: lineStart, to: cursor)
}
/// One visual line up. Returns caret offset and the display column to
/// keep sticky for the rest of this vertical drag.
public static func visualLineUpOffset(
before: String?,
after: String?,
preferredDisplayColumn: CGFloat?,
config: VisualLineLayoutConfig
) -> (offset: Int, stickyColumn: CGFloat)? {
let text = mergedContext(before: before, after: after)
let cursor = before?.count ?? 0
let layout = VisualLineLayout(text: text, config: config)
guard let currentLine = layout.lineIndex(containing: cursor), currentLine > 0 else {
return nil
}
let sticky = preferredDisplayColumn
?? layout.width(from: layout.lineStarts[currentLine], to: cursor)
let previousStart = layout.lineStarts[currentLine - 1]
let previousEnd = layout.lineStarts[currentLine]
let target = layout.offset(
onLineStartingAt: previousStart,
lineEndingBefore: previousEnd,
displayColumn: sticky
)
let offset = target - cursor
guard offset != 0 else { return nil }
return (offset, sticky)
}
/// One visual line down.
public static func visualLineDownOffset(
before: String?,
after: String?,
preferredDisplayColumn: CGFloat?,
config: VisualLineLayoutConfig
) -> (offset: Int, stickyColumn: CGFloat)? {
let text = mergedContext(before: before, after: after)
let cursor = before?.count ?? 0
let layout = VisualLineLayout(text: text, config: config)
guard let currentLine = layout.lineIndex(containing: cursor) else { return nil }
guard currentLine + 1 < layout.lineStarts.count else { return nil }
let sticky = preferredDisplayColumn
?? layout.width(from: layout.lineStarts[currentLine], to: cursor)
let nextStart = layout.lineStarts[currentLine + 1]
let nextEnd = currentLine + 2 < layout.lineStarts.count
? layout.lineStarts[currentLine + 2]
: text.count
let target = layout.offset(
onLineStartingAt: nextStart,
lineEndingBefore: nextEnd,
displayColumn: sticky
)
let offset = target - cursor
guard offset != 0 else { return nil }
return (offset, sticky)
}
// MARK: - Default width table
/// Crude fallback advance width: wide scripts count double, everything
/// else single. Used by tests and when real metrics are unavailable.
public static func defaultDisplayWidth(_ character: Character) -> CGFloat {
guard let scalar = character.unicodeScalars.first else { return 1 }
if character == "\n" { return 0 }
if character == "\t" { return 4 }
if isWide(scalar) { return 2 }
return 1
}
private static func isWide(_ scalar: UnicodeScalar) -> Bool {
let value = scalar.value
return (0x1100...0x115F).contains(value) // Hangul Jamo
|| (0x2E80...0xA4CF).contains(value) // CJK radicals, symbols, bopomofo, yi
|| (0xAC00...0xD7A3).contains(value) // Hangul syllables
|| (0xF900...0xFAFF).contains(value) // CJK compatibility
|| (0xFE10...0xFE1F).contains(value) // vertical forms
|| (0xFE30...0xFE6F).contains(value) // CJK compatibility forms
|| (0xFF00...0xFF60).contains(value) // fullwidth
|| (0xFFE0...0xFFE6).contains(value) // fullwidth symbols
|| (0x20000...0x2FFFF).contains(value) // CJK extension planes
|| (0x30000...0x3FFFF).contains(value)
}
// MARK: - Internals
private static func mergedContext(before: String?, after: String?) -> String {
(before ?? "") + (after ?? "")
}
// MARK: - Visual line layout
struct VisualLineLayout {
let text: String
let widthOf: CharacterWidth
let lineStarts: [Int]
init(text: String, config: VisualLineLayoutConfig) {
self.text = text
self.widthOf = config.widthOf
self.lineStarts = Self.computeLineStarts(
in: text,
maxWidth: config.lineWidth,
widthOf: config.widthOf
)
}
func lineIndex(containing offset: Int) -> Int? {
guard !lineStarts.isEmpty else { return nil }
for index in lineStarts.indices.reversed() where offset >= lineStarts[index] {
return index
}
return nil
}
func lineStart(containing offset: Int) -> Int {
lineIndex(containing: offset).map { lineStarts[$0] } ?? 0
}
func width(from start: Int, to end: Int) -> CGFloat {
guard start < end, end <= text.count else { return 0 }
let startIndex = text.index(text.startIndex, offsetBy: start)
let endIndex = text.index(text.startIndex, offsetBy: end)
var total: CGFloat = 0
var index = startIndex
while index < endIndex {
total += widthOf(text[index])
index = text.index(after: index)
}
return total
}
func offset(
onLineStartingAt lineStart: Int,
lineEndingBefore lineEnd: Int,
displayColumn: CGFloat
) -> Int {
guard lineStart <= lineEnd, lineEnd <= text.count else { return lineStart }
let startIndex = text.index(text.startIndex, offsetBy: lineStart)
let endIndex = text.index(text.startIndex, offsetBy: lineEnd)
var total: CGFloat = 0
var index = startIndex
while index < endIndex {
let advance = widthOf(text[index])
if total + advance > displayColumn { break }
total += advance
index = text.index(after: index)
}
return text.distance(from: text.startIndex, to: index)
}
private static func computeLineStarts(
in text: String,
maxWidth: CGFloat,
widthOf: CharacterWidth
) -> [Int] {
guard !text.isEmpty else { return [0] }
var starts: [Int] = [0]
var lineWidth: CGFloat = 0
var lineStart = text.startIndex
var lastBreak: String.Index?
var index = text.startIndex
while index < text.endIndex {
let character = text[index]
if character == "\n" {
let next = text.index(after: index)
let nextOffset = text.distance(from: text.startIndex, to: next)
if starts.last != nextOffset {
starts.append(nextOffset)
}
lineStart = next
lineWidth = 0
lastBreak = nil
index = next
continue
}
let advance = widthOf(character)
if character == " " || character == "\t" {
lastBreak = index
}
if lineWidth + advance > maxWidth, index > lineStart {
let breakIndex: String.Index
if let lastBreak, lastBreak > lineStart {
breakIndex = text.index(after: lastBreak)
} else {
breakIndex = index
}
let breakOffset = text.distance(from: text.startIndex, to: breakIndex)
if starts.last != breakOffset {
starts.append(breakOffset)
}
lineStart = breakIndex
lineWidth = 0
lastBreak = nil
if breakIndex == index {
lineWidth = advance
index = text.index(after: index)
}
continue
}
lineWidth += advance
index = text.index(after: index)
}
return starts
}
}
}
#if canImport(UIKit)
import UIKit
/// Real-font per-character advance widths (in points) for cursor visual-line
/// navigation. Caches measurements so repeated drag samples are cheap.
///
/// Absolute values assume a ~17 pt body font; only the *ratios* between
/// glyphs (and between a glyph and the field width) matter for column
/// fidelity, so a reference font is sufficient to eliminate the fixed-width
/// column drift.
///
/// Not actor-isolated on purpose: the width closure is invoked synchronously
/// from the nonisolated `CursorNavigation` layout code. A lock guards the
/// cache so `@unchecked Sendable` is safe.
public final class CursorGlyphMetrics: @unchecked Sendable {
public static let shared = CursorGlyphMetrics()
private let font = UIFont.systemFont(ofSize: 17)
private let lock = NSLock()
private var cache: [Character: CGFloat] = [:]
public init() {}
public func width(of character: Character) -> CGFloat {
if character == "\n" { return 0 }
lock.lock()
defer { lock.unlock() }
if let cached = cache[character] { return cached }
let measured = (String(character) as NSString)
.size(withAttributes: [.font: font])
.width
let width = measured > 0 ? measured : font.pointSize * 0.5
cache[character] = width
return width
}
}
#endif
OSGKeyboardShared/Services/KeyboardState.swift
+14 -12
View File
@@ -134,8 +134,6 @@ public final class KeyboardState: ObservableObject {
/// Mirrors the host field's return-key intent. The action stays a newline
/// insert; host apps decide whether that submits or creates a line break.
@Published public var returnKeyRole: ReturnKeyRole = .newline
/// Press-and-drag pads beside the mic for four-way caret movement.
@Published public var cursorDragNavigationEnabled: Bool = true
/// Opt-in clipboard history capture (mirrored from App Group).
@Published public var clipboardHistoryEnabled: Bool = false
/// Opt-in clipboard suggestion strip (requires history enabled).
@@ -174,12 +172,11 @@ public final class KeyboardState: ObservableObject {
/// constraint and the SwiftUI key grid pick their metrics from this one
/// value so they can never disagree and clip the bottom row.
@Published public var layoutWidth: CGFloat = 0
/// `true` while a cursor-drag pad is being pressed — drives the hint
/// shown above the mic.
@Published public var cursorDragActive: Bool = false
/// `true` when the last voice insertion is still at the caret and can
/// be undone (suffix-checked against `documentContextBeforeInput`).
@Published public var undoAvailable: Bool = false
/// A verified OSG-generated insertion can be edited by voice.
@Published public var editAvailable: Bool = false
/// `true` while an undone voice insertion can be re-applied (redo buffer).
@Published public var redoAvailable: Bool = false
/// `true` when the host field has a non-empty selection (copy enabled).
@@ -190,6 +187,11 @@ public final class KeyboardState: ObservableObject {
@Published public var editSession: EditSessionState = .inactive
/// AI conversation UI state for the keyboard surface. The host owns the actual messages.
@Published public var aiSession: AISessionState = .inactive
/// The latest generated insertion can be submitted through the host's
/// action-style Return key (Send / Search / Done / Go).
@Published public var assistantSendAvailable: Bool = false
/// Brief success pulse rendered on the unified assistant microphone.
@Published public var assistantInsertionSucceeded: Bool = false
@Published public var editCanReplaceOriginal: Bool = false
/// Short idle feedback (availability, expiry, missing LLM).
@Published public var editHint: String?
@@ -210,6 +212,7 @@ public final class KeyboardState: ObservableObject {
clipboardSuggestionText = nil
clipboardSuggestionChangeCount = nil
clipboardOverlay = .none
assistantSendAvailable = false
}
// MARK: - Host-app onboarding gate
@@ -270,7 +273,11 @@ public final class KeyboardState: ObservableObject {
public var closeEditMode: () -> Void = {}
public var tapAIMic: () -> Void = {}
public var cancelAIInput: () -> Void = {}
public var sendAIAnswer: () -> Void = {}
/// Explicitly inserts a retained AI result after target validation failed.
public var confirmPendingAIAnswer: () -> Void = {}
public var discardPendingAIAnswer: () -> Void = {}
/// Performs the host's action-style Return after generated text was inserted.
public var sendAssistantAction: () -> Void = {}
/// Sends a tapped idle hint card as the AI question (skip microphone).
public var submitAIHint: (AIHintCard) -> Void = { _ in }
/// Sends a clipboard skill (reply / summarize / translate / export).
@@ -315,11 +322,6 @@ public final class KeyboardState: ObservableObject {
public var copySelection: () -> Void = {}
/// Cut the current text selection (copy + delete).
public var cutSelection: () -> Void = {}
public var moveCursorHorizontal: (Int) -> Void = { _ in }
public var moveCursorVertical: (Int) -> Void = { _ in }
/// Cursor-drag pad press lifecycle — updates `cursorDragActive` and
/// lets the view controller reset vertical-navigation stickiness.
public var setCursorDragActive: (Bool) -> Void = { _ in }
/// Switch voice ↔ typing. No-ops when voice pipeline is active.
public var setSurface: (Surface) -> Void = { _ in }
@@ -338,7 +340,7 @@ public final class KeyboardState: ObservableObject {
public var canEnterTypingSurface: Bool { !locksTypingSurface }
public var canCancelAIInput: Bool {
surface == .ai && aiSession.isBusy
aiSession.isBusy
}
/// Normal dictation can be discarded from microphone startup through