rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(keyboard): add Notes skill and system-style typing

Browse Source 
Add save-to-Notes export and direct map handoff while aligning multi-touch typing, period shortcuts, return actions, and navigation visuals with system behavior.
This commit is contained in:
Rocky
2026-08-14 18:22:41 +08:00
parent 0a60006d2d
commit 320dc777c9
46 changed files with 1429 additions and 376 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OSGKeyboardExt/KeyboardViewController.swift
+23 -3
View File
@@ -773,10 +773,30 @@ public final class KeyboardViewController: UIInputViewController {
private func returnKeyRole(for returnKeyType: UIReturnKeyType) -> State.ReturnKeyRole {
switch returnKeyType {
case .send, .go, .search, .join, .route, .google, .yahoo, .continue, .emergencyCall:
return .send
case .default, .next, .done:
case .default:
return .newline
case .go:
return .go
case .google:
return .google
case .join:
return .join
case .next:
return .next
case .route:
return .route
case .search:
return .search
case .send:
return .send
case .yahoo:
return .yahoo
case .done:
return .done
case .emergencyCall:
return .emergencyCall
case .continue:
return .continue
@unknown default:
return .newline
}
OSGKeyboardExt/Services/AIKeyboardCoordinator.swift
+45 -16
View File
@@ -70,13 +70,18 @@ final class AIKeyboardCoordinator {
func submitClipboardSkill(_ skill: AIClipboardSkill) {
guard canAcceptIdleSubmit else { return }
enterIfNeeded()
state.pendingClipboardSkillID = skill.kind == .export ? skill.id : nil
let material = ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
if skill.kind == .export {
state.pendingClipboardSkillID = skill.id
state.pendingClipboardSkillSource = material
} else {
clearPendingExportSkill()
}
let instruction = AIClipboardSkillCatalog.instruction(
for: skill,
locale: AIHintLocaleResolver.packLocale(),
translationTargetLocaleId: state.translationTargetLocaleId
)
let material = ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
AIAgentShortcutRun.trace("keyboard.submit skill=\(skill.id) kind=\(skill.kind)")
if let material {
AIAgentShortcutRun.traceBody("keyboard.clipboard", material)
@@ -120,7 +125,7 @@ final class AIKeyboardCoordinator {
) {
guard case .ready(let prompt) = resolution else {
// The clipboard window closed between rendering and this tap.
state.pendingClipboardSkillID = nil
clearPendingExportSkill()
state.aiSession.fail(
ExtL10n.string("keyboard.ai.error.clipboardUnavailable"),
utteranceID: nil
@@ -128,7 +133,7 @@ final class AIKeyboardCoordinator {
return
}
guard let conversationID = state.aiSession.conversationID else {
state.pendingClipboardSkillID = nil
clearPendingExportSkill()
return
}
let disposition = flow.submitAIQuestion(
@@ -137,14 +142,14 @@ final class AIKeyboardCoordinator {
thinkingEnabled: thinkingEnabled
)
if case .rejected(let rejection) = disposition {
state.pendingClipboardSkillID = nil
clearPendingExportSkill()
state.aiSession.fail(message(for: rejection), utteranceID: nil)
}
}
func cancel() {
guard state.aiSession.isBusy else { return }
state.pendingClipboardSkillID = nil
clearPendingExportSkill()
flow.cancelAIRecording()
state.aiSession.cancelCurrentWork()
}
@@ -153,7 +158,7 @@ final class AIKeyboardCoordinator {
if state.aiSession.canInsert, let answer = state.aiSession.answer {
guard insertAnswer(answer) else { return }
state.aiSession.markAnswerInserted(
offersSend: state.returnKeyRole == .send
offersSend: state.returnKeyRole.usesActionFill
)
} else if state.aiSession.canSend {
state.aiSession.markAnswerSent()
@@ -228,16 +233,21 @@ final class AIKeyboardCoordinator {
}
func fail(_ message: String, utteranceID: UUID?) {
state.pendingClipboardSkillID = nil
clearPendingExportSkill()
state.aiSession.fail(message, utteranceID: utteranceID)
}
private func endConversationIfNeeded() {
state.pendingClipboardSkillID = nil
clearPendingExportSkill()
guard let conversationID = state.aiSession.conversationID else { return }
flow.endAIConversation(conversationID)
}
private func clearPendingExportSkill() {
state.pendingClipboardSkillID = nil
state.pendingClipboardSkillSource = nil
}
private var isPendingExportSkill: Bool {
guard let id = state.pendingClipboardSkillID else { return false }
return resolvedSkill(id: id)?.kind == .export
@@ -251,9 +261,10 @@ final class AIKeyboardCoordinator {
}
/// Parse an export skill. Empty → in-keyboard tip, stay in the host app.
/// Lines → hand off to the host to run the companion Shortcut.
/// Lines → hand off to the host (Shortcut, Maps, or Didi).
private func finishExportSkill(answer: String) {
let source = ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
let source = state.pendingClipboardSkillSource
?? ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
let skillID = state.pendingClipboardSkillID
let items: [String]
let emptyTipKey: String
@@ -267,6 +278,13 @@ final class AIKeyboardCoordinator {
case AIClipboardSkillCatalog.navigateID:
items = AIAddressExtraction.lines(from: answer, sourceClipboard: source)
emptyTipKey = "keyboard.ai.skill.noAddress"
case AIClipboardSkillCatalog.saveToNotesID:
items = AINoteExport.items(
from: answer,
sourceClipboard: source,
locale: AIHintLocaleResolver.packLocale()
)
emptyTipKey = "keyboard.ai.skill.noNote"
default:
items = AIGenericSkillExport.items(from: answer)
emptyTipKey = "keyboard.ai.skill.noExportItems"
@@ -280,7 +298,7 @@ final class AIKeyboardCoordinator {
AIAgentShortcutRun.traceBody("keyboard.parsedTitles", items.joined(separator: "\n"))
}
#endif
state.pendingClipboardSkillID = nil
clearPendingExportSkill()
if state.aiSession.isBusy {
state.aiSession.cancelCurrentWork()
}
@@ -292,14 +310,25 @@ final class AIKeyboardCoordinator {
state.skillTipText = ExtL10n.string(emptyTipKey)
return
}
guard let skillID,
resolvedSkill(id: skillID)?.shortcutName != nil else {
AIAgentShortcutRun.trace("keyboard.parse missingShortcut skill=\(skillID ?? "nil")")
guard let skillID, let skill = resolvedSkill(id: skillID) else {
AIAgentShortcutRun.trace("keyboard.parse missingSkill")
state.skillTipText = ExtL10n.string("keyboard.ai.skill.shortcutMissing")
return
}
if skill.requiresShortcut, skill.shortcutName == nil {
AIAgentShortcutRun.trace("keyboard.parse missingShortcut skill=\(skillID)")
state.skillTipText = ExtL10n.string("keyboard.ai.skill.shortcutMissing")
return
}
AIAgentShortcutRun.trace("keyboard.handoffToHost skill=\(skillID) items=\(items.count)")
state.skillTipText = ExtL10n.string("keyboard.ai.skill.runningShortcut")
let tipKey: String
switch skillID {
case AIClipboardSkillCatalog.navigateID:
tipKey = "keyboard.ai.skill.openingMaps"
default:
tipKey = "keyboard.ai.skill.runningShortcut"
}
state.skillTipText = ExtL10n.string(tipKey)
state.runClipboardExportSkill(skillID, items)
}
OSGKeyboardExt/Typing/TypingKeyTouchPad.swift
+51 -98
View File
@@ -3,7 +3,8 @@
//
// Grid-level UIKit touch tracking for the typing surface:
// Down highlight → Move reselect → Up commit (letters / space / return),
// delete repeats on down, Shift holds while the gesture owns it.
// overlapping fingers commit in press order, delete repeats on down,
// Shift holds while that finger owns it.
import SwiftUI
import UIKit
@@ -12,7 +13,7 @@ import OSGKeyboardShared
struct TypingKeyTouchPad: UIViewRepresentable {
var layout: TypingKeyLayout
var hapticIntensity: KeyboardHapticIntensity
var onHighlightChange: (String?) -> Void
var onHighlightChange: (Set<String>) -> Void
var onCommit: (TypingKeyHitTarget) -> Void
var onDeleteFire: () -> Void
var onShiftBegan: () -> Void
@@ -36,8 +37,7 @@ struct TypingKeyTouchPad: UIViewRepresentable {
@MainActor
final class Coordinator {
var parent: TypingKeyTouchPad
private var activeKeyID: String?
private var gestureOwnsShift = false
private let tracker = TypingTouchTracker()
private var deleteRepeatTask: Task<Void, Never>?
private var deleteRepeatStartedAt: Date?
private var isDeleteRepeating = false
@@ -46,72 +46,24 @@ struct TypingKeyTouchPad: UIViewRepresentable {
self.parent = parent
}
func handleBegan(at point: CGPoint) {
resetDeleteRepeat()
gestureOwnsShift = false
guard let key = hit(at: point) else {
setHighlight(nil)
return
}
activate(key)
func handleBegan(id: ObjectIdentifier, at point: CGPoint) {
apply(tracker.began(id: id, key: hit(at: point)))
}
func handleMoved(at point: CGPoint) {
let key = hit(at: point)
if key?.id == activeKeyID { return }
if activeBehavior == .deleteRepeat {
stopDeleteRepeat()
}
if let key {
activate(key)
} else {
// Outside plane: clear highlight; Shift stays held until ended.
setHighlight(nil)
activeKeyID = nil
}
func handleMoved(id: ObjectIdentifier, at point: CGPoint) {
apply(tracker.moved(id: id, key: hit(at: point)))
}
func handleEnded(at point: CGPoint) {
// Outside the key plane → cancel (no commit), per accuracy plan.
let key = hit(at: point)
stopDeleteRepeat()
defer {
setHighlight(nil)
activeKeyID = nil
finishShiftIfNeeded()
}
guard let key else { return }
switch key.behavior {
case .commitOnRelease:
parent.onCommit(key)
case .deleteRepeat:
// Already fired on down / while held.
break
case .shiftHold:
// endShiftHold decides tap vs hold-with-type.
break
}
func handleEnded(id: ObjectIdentifier, at point: CGPoint) {
apply(tracker.ended(id: id, key: hit(at: point)))
}
func handleCancelled() {
stopDeleteRepeat()
setHighlight(nil)
activeKeyID = nil
finishShiftIfNeeded()
func handleCancelled(id: ObjectIdentifier) {
apply(tracker.cancelled(id: id))
}
// MARK: - Internals
private var activeBehavior: TypingKeyTouchBehavior? {
guard let activeKeyID else { return nil }
return parent.layout.key(id: activeKeyID)?.behavior
}
private func hit(at point: CGPoint) -> TypingKeyHitTarget? {
let layout = parent.layout
return KeyHitTesting.hitTarget(
@@ -124,27 +76,29 @@ struct TypingKeyTouchPad: UIViewRepresentable {
)
}
private func activate(_ key: TypingKeyHitTarget) {
activeKeyID = key.id
setHighlight(key.id)
playFeedback(for: key)
switch key.behavior {
case .commitOnRelease:
break
case .deleteRepeat:
parent.onDeleteFire()
startDeleteRepeat()
case .shiftHold:
if !gestureOwnsShift {
gestureOwnsShift = true
parent.onShiftBegan()
}
private func apply(_ effects: TypingTouchEffects) {
if effects.stopDeleteRepeat {
stopDeleteRepeat()
}
}
private func setHighlight(_ id: String?) {
parent.onHighlightChange(id)
for key in effects.commits {
parent.onCommit(key)
}
if let key = effects.playFeedback {
playFeedback(for: key)
}
if effects.deleteFire {
parent.onDeleteFire()
}
if effects.startDeleteRepeat {
startDeleteRepeat()
}
if effects.beginShift {
parent.onShiftBegan()
}
if effects.endShift {
parent.onShiftEnded()
}
parent.onHighlightChange(tracker.highlightedKeyIDs)
}
private func playFeedback(for key: TypingKeyHitTarget) {
@@ -207,16 +161,6 @@ struct TypingKeyTouchPad: UIViewRepresentable {
deleteRepeatTask?.cancel()
deleteRepeatTask = nil
}
private func resetDeleteRepeat() {
stopDeleteRepeat()
}
private func finishShiftIfNeeded() {
guard gestureOwnsShift else { return }
gestureOwnsShift = false
parent.onShiftEnded()
}
}
}
@@ -239,7 +183,7 @@ final class TypingKeyTouchPadUIView: UIView {
override init(frame: CGRect) {
super.init(frame: frame)
backgroundColor = Self.padTint
isMultipleTouchEnabled = false
isMultipleTouchEnabled = true
isExclusiveTouch = true
isUserInteractionEnabled = true
}
@@ -250,21 +194,30 @@ final class TypingKeyTouchPadUIView: UIView {
}
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
guard let touch = touches.first else { return }
coordinator?.handleBegan(at: touch.location(in: self))
for touch in Self.sorted(touches) {
coordinator?.handleBegan(id: ObjectIdentifier(touch), at: touch.location(in: self))
}
}
override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
guard let touch = touches.first else { return }
coordinator?.handleMoved(at: touch.location(in: self))
for touch in Self.sorted(touches) {
coordinator?.handleMoved(id: ObjectIdentifier(touch), at: touch.location(in: self))
}
}
override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) {
guard let touch = touches.first else { return }
coordinator?.handleEnded(at: touch.location(in: self))
for touch in Self.sorted(touches) {
coordinator?.handleEnded(id: ObjectIdentifier(touch), at: touch.location(in: self))
}
}
override func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent?) {
coordinator?.handleCancelled()
for touch in Self.sorted(touches) {
coordinator?.handleCancelled(id: ObjectIdentifier(touch))
}
}
private static func sorted(_ touches: Set<UITouch>) -> [UITouch] {
touches.sorted { $0.timestamp < $1.timestamp }
}
}
OSGKeyboardExt/Typing/TypingRootView.swift
+21 -33
View File
@@ -47,8 +47,8 @@ struct TypingRootView: View {
/// After the first expand, keep the panel tree mounted and only toggle
/// opacity so subsequent ▼/▲ taps stay cheap.
@State private var candidatePanelMounted = false
/// Key currently under the finger (grid-level touch pad).
@State private var highlightedKeyID: String?
/// Keys currently under a finger (grid-level touch pad, multi-touch).
@State private var highlightedKeyIDs: Set<String> = []
static func totalHeight(isIPad: Bool = false, width: CGFloat = 0) -> CGFloat {
TypingLayoutMetrics.contentHeight(isIPad: isIPad, width: width)
@@ -398,7 +398,7 @@ struct TypingRootView: View {
TypingKeyTouchPad(
layout: layout,
hapticIntensity: state.keyboardHapticIntensity,
onHighlightChange: { highlightedKeyID = $0 },
onHighlightChange: { highlightedKeyIDs = $0 },
onCommit: { commitTypingKey($0) },
onDeleteFire: { apply(typing.handleKey("⌫")) },
onShiftBegan: { typing.beginShiftHold() },
@@ -448,10 +448,10 @@ struct TypingRootView: View {
let pageLabel = typing.page == .letters ? "123" : "ABC"
let spaceLabel = typing.language == .chinese ? "空格" : "space"
let returnLabel: String = {
switch state.returnKeyRole {
case .newline: return "return"
case .send: return ExtL10n.string(state.returnKeyRole.titleKey)
if state.returnKeyRole.usesActionFill {
return ExtL10n.string(state.returnKeyRole.titleKey)
}
return "return"
}()
// iPad spends its extra width on comma / period like the system
@@ -491,7 +491,7 @@ struct TypingRootView: View {
@ViewBuilder
private func visualTypingKey(_ key: TypingKeyHitTarget) -> some View {
let pressed = highlightedKeyID == key.id
let pressed = highlightedKeyIDs.contains(key.id)
let isBottom = key.id.hasPrefix("bottom.")
let isShift = key.label == "⇧"
let shiftLit = isShift && typing.isShiftEnabled
@@ -624,13 +624,14 @@ struct TypingRootView: View {
@ViewBuilder
private var returnKeyLabel: some View {
switch state.returnKeyRole {
case .newline:
Image(systemName: "arrow.turn.down.left")
.font(.system(size: 21, weight: .medium))
case .send:
if state.returnKeyRole.usesActionFill {
ExtL10n.text(state.returnKeyRole.titleKey)
.font(.system(size: 15, weight: .semibold))
.lineLimit(1)
.minimumScaleFactor(0.6)
} else {
Image(systemName: "arrow.turn.down.left")
.font(.system(size: 21, weight: .medium))
}
}
@@ -684,42 +685,29 @@ struct TypingRootView: View {
}
private var returnKeyFill: Color {
switch state.returnKeyRole {
case .newline: return keyFill
case .send: return sendKeyFill
}
state.returnKeyRole.usesActionFill ? sendKeyFill : keyFill
}
private var returnKeyPressedFill: Color {
switch state.returnKeyRole {
case .newline: return keyPressedFill
case .send: return sendKeyPressedFill
}
state.returnKeyRole.usesActionFill ? sendKeyPressedFill : keyPressedFill
}
private var returnKeyBorder: Color {
switch state.returnKeyRole {
case .newline:
return palette.divider
case .send:
if state.returnKeyRole.usesActionFill {
return Color.black.opacity(colorScheme == .dark ? 0.10 : 0.08)
}
return palette.divider
}
private var returnKeyAccessibilityLabel: String {
switch state.returnKeyRole {
case .newline: return "return"
case .send: return ExtL10n.string(state.returnKeyRole.titleKey)
if state.returnKeyRole.usesActionFill {
return ExtL10n.string(state.returnKeyRole.titleKey)
}
return "return"
}
private var returnKeyTextColor: Color {
switch state.returnKeyRole {
case .newline:
return keyTextColor
case .send:
return .white
}
state.returnKeyRole.usesActionFill ? .white : keyTextColor
}
/// The send key stays recognizable in both appearances without becoming neon.
OSGKeyboardExt/Views/AIKeyboardView.swift
+3
View File
@@ -518,6 +518,9 @@ struct AIKeyboardView: View {
if state.pendingClipboardSkillID == AIClipboardSkillCatalog.navigateID {
return ExtL10n.string("keyboard.ai.skill.extractingAddress")
}
if state.pendingClipboardSkillID == AIClipboardSkillCatalog.saveToNotesID {
return ExtL10n.string("keyboard.ai.skill.namingNote")
}
if let draft = state.aiSession.draftAnswerText, !draft.isEmpty {
return ExtL10n.string("keyboard.ai.generating")
}
OSGKeyboardExt/Views/KeyboardRootView.swift
+1 -1
View File
@@ -484,7 +484,7 @@ public struct KeyboardRootView: View {
title: title,
label: title,
disabled: disabled,
isSend: state.returnKeyRole == .send,
isSend: state.returnKeyRole.usesActionFill,
hapticIntensity: state.keyboardHapticIntensity
) {
state.insertNewline()
OSGKeyboardExt/en.lproj/Keyboard.strings
+12
View File
@@ -26,6 +26,14 @@
"common.space" = "Space";
"common.newline" = "Return";
"common.send" = "Send";
"keyboard.return.go" = "Go";
"keyboard.return.search" = "Search";
"keyboard.return.join" = "Join";
"keyboard.return.next" = "Next";
"keyboard.return.route" = "Route";
"keyboard.return.google" = "Google";
"keyboard.return.yahoo" = "Yahoo";
"keyboard.return.emergencyCall" = "Emergency Call";
/* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "On-device transcription";
@@ -299,6 +307,7 @@
"keyboard.ai.skill.translate" = "Translate";
"keyboard.ai.skill.extractTodos" = "Tasks";
"keyboard.ai.skill.extractEvents" = "Events";
"keyboard.ai.skill.saveToNotes" = "Notes";
"keyboard.ai.skill.navigate" = "Navigate";
"keyboard.ai.skill.previewPolish" = "Polish";
"keyboard.ai.skill.previewIdeas" = "Ideas";
@@ -306,10 +315,13 @@
"keyboard.ai.skill.noTodos" = "No tasks in the clipboard";
"keyboard.ai.skill.noEvents" = "No date or time in the clipboard";
"keyboard.ai.skill.noAddress" = "No address in the clipboard";
"keyboard.ai.skill.noNote" = "Nothing to save to Notes";
"keyboard.ai.skill.noExportItems" = "Nothing to send to the Shortcut";
"keyboard.ai.skill.extracting" = "Finding tasks…";
"keyboard.ai.skill.extractingEvents" = "Finding events…";
"keyboard.ai.skill.extractingAddress" = "Finding an address…";
"keyboard.ai.skill.namingNote" = "Naming the note…";
"keyboard.ai.skill.runningShortcut" = "Running Shortcut…";
"keyboard.ai.skill.openingMaps" = "Opening Maps…";
"keyboard.ai.skill.shortcutMissing" = "Companion Shortcut not found. Reinstall it in Skills";
"keyboard.ai.skill.handoffFailed" = "Couldn’t open the app to run the Shortcut";
OSGKeyboardExt/zh-Hans.lproj/Keyboard.strings
+12
View File
@@ -26,6 +26,14 @@
"common.space" = "空格";
"common.newline" = "回车";
"common.send" = "发送";
"keyboard.return.go" = "前往";
"keyboard.return.search" = "搜索";
"keyboard.return.join" = "加入";
"keyboard.return.next" = "下一项";
"keyboard.return.route" = "路线";
"keyboard.return.google" = "Google";
"keyboard.return.yahoo" = "Yahoo";
"keyboard.return.emergencyCall" = "紧急电话";
/* Privacy footnote (onboarding welcome page) */
"privacy.audio.title" = "支持本地转写";
@@ -299,6 +307,7 @@
"keyboard.ai.skill.translate" = "翻译";
"keyboard.ai.skill.extractTodos" = "待办";
"keyboard.ai.skill.extractEvents" = "日程";
"keyboard.ai.skill.saveToNotes" = "备忘录";
"keyboard.ai.skill.navigate" = "导航";
"keyboard.ai.skill.previewPolish" = "润色";
"keyboard.ai.skill.previewIdeas" = "点子";
@@ -306,10 +315,13 @@
"keyboard.ai.skill.noTodos" = "剪贴板内容没有待办事项";
"keyboard.ai.skill.noEvents" = "剪贴板内容没有日期或时间";
"keyboard.ai.skill.noAddress" = "剪贴板内容没有可导航的地址";
"keyboard.ai.skill.noNote" = "剪贴板没有可保存的内容";
"keyboard.ai.skill.noExportItems" = "没有可发给捷径的内容";
"keyboard.ai.skill.extracting" = "正在提取待办…";
"keyboard.ai.skill.extractingEvents" = "正在提取日程…";
"keyboard.ai.skill.extractingAddress" = "正在识别地址…";
"keyboard.ai.skill.namingNote" = "正在生成标题…";
"keyboard.ai.skill.runningShortcut" = "正在运行捷径…";
"keyboard.ai.skill.openingMaps" = "正在打开地图…";
"keyboard.ai.skill.shortcutMissing" = "找不到配套捷径，请在技能页重新安装";
"keyboard.ai.skill.handoffFailed" = "无法打开 App 运行捷径";