feat(keyboard): add clipboard voice command mode

Long-press mic runs ASR as an instruction over eligible clipboard text,
with host-confirm recording UI, min-record gate, and light ASR prewarm.
Bump CURRENT_PROJECT_VERSION to 53.
This commit is contained in:
Rocky
2026-08-07 16:08:42 +08:00
parent 98ae211b3d
commit f197b68573
22 changed files with 1362 additions and 124 deletions
@@ -24,21 +24,29 @@ public struct RecordButton: View {
public let remainingSeconds: Int?
public let isEnabled: Bool
public let onToggle: () -> Void
/// When non-nil, a 0.45s hold starts clipboard-command recording instead of toggle.
public let onClipboardLongPressBegan: (() -> Void)?
public let onClipboardLongPressEnded: (() -> Void)?
@State private var breath = false
@State private var longPressArmed = false
public init(
phase: Phase,
level: Double,
remainingSeconds: Int? = nil,
isEnabled: Bool = true,
onToggle: @escaping () -> Void
onToggle: @escaping () -> Void,
onClipboardLongPressBegan: (() -> Void)? = nil,
onClipboardLongPressEnded: (() -> Void)? = nil
) {
self.phase = phase
self.level = level
self.remainingSeconds = remainingSeconds
self.isEnabled = isEnabled
self.onToggle = onToggle
self.onClipboardLongPressBegan = onClipboardLongPressBegan
self.onClipboardLongPressEnded = onClipboardLongPressEnded
}
private var isUrgent: Bool {
@@ -131,14 +139,23 @@ public struct RecordButton: View {
.animation(Motion.soft, value: remainingSeconds)
}
.contentShape(Circle())
.onTapGesture {
guard phase != .processing else { return }
guard isEnabled || phase == .idleUnavailable else { return }
onToggle()
}
.modifier(
RecordButtonPressModifier(
phase: phase,
isEnabled: isEnabled,
supportsClipboardLongPress: onClipboardLongPressBegan != nil,
longPressArmed: $longPressArmed,
onToggle: onToggle,
onClipboardLongPressBegan: onClipboardLongPressBegan,
onClipboardLongPressEnded: onClipboardLongPressEnded
)
)
.onAppear { breath = (phase == .recording) }
.onChange(of: phase) { _, new in
breath = (new == .recording)
if new != .recording {
longPressArmed = false
}
}
.accessibilityLabel(Text(SharedL10n.string("keyboard.tapToTalkA11y")))
}
@@ -186,3 +203,55 @@ public struct RecordButton: View {
}
}
}
// MARK: - Press / long-press routing
private struct RecordButtonPressModifier: ViewModifier {
let phase: RecordButton.Phase
let isEnabled: Bool
let supportsClipboardLongPress: Bool
@Binding var longPressArmed: Bool
let onToggle: () -> Void
let onClipboardLongPressBegan: (() -> Void)?
let onClipboardLongPressEnded: (() -> Void)?
func body(content: Content) -> some View {
if supportsClipboardLongPress {
content.onLongPressGesture(
minimumDuration: ClipboardMaterialFilter.longPressDuration,
maximumDistance: 120,
pressing: { pressing in
if pressing {
longPressArmed = false
return
}
// Released.
if longPressArmed {
onClipboardLongPressEnded?()
longPressArmed = false
} else if phase != .processing, isEnabled || phase == .idleUnavailable {
// Short press existing tap-toggle dictation.
guard phase != .recording else {
// If somehow recording without arming, end via toggle.
onToggle()
return
}
onToggle()
}
},
perform: {
guard phase != .processing else { return }
guard isEnabled || phase == .idleUnavailable else { return }
longPressArmed = true
onClipboardLongPressBegan?()
}
)
} else {
content.onTapGesture {
guard phase != .processing else { return }
guard isEnabled || phase == .idleUnavailable else { return }
onToggle()
}
}
}
}
@@ -0,0 +1,14 @@
// FlowUtteranceMode.swift
// OSGKeyboard · Shared
//
// Distinguishes dictation polish from clipboard-command generation on the
// Flow command / result wire (plan §11).
import Foundation
public enum FlowUtteranceMode: String, Codable, Equatable, Sendable {
/// ASR is draft text to polish and insert (default / legacy).
case dictation
/// ASR is an instruction over a frozen clipboard snapshot.
case clipboardCommand
}
@@ -0,0 +1,96 @@
// ClipboardCommandEligibility.swift
// OSGKeyboard · Shared
//
// Tracks clipboard open-window eligibility (30s from first sighting of a
// pasteboard change) for opportunity-read UI. Pure timing logic no UIKit.
import Foundation
public struct ClipboardCommandEligibility: Equatable, Sendable {
public var changeCount: Int
public var snapshot: String
public var startedAt: TimeInterval
public init(changeCount: Int, snapshot: String, startedAt: TimeInterval = Date().timeIntervalSince1970) {
self.changeCount = changeCount
self.snapshot = snapshot
self.startedAt = startedAt
}
public func isOpen(at now: TimeInterval = Date().timeIntervalSince1970) -> Bool {
now - startedAt <= ClipboardMaterialFilter.eligibilityDuration
}
public func remaining(at now: TimeInterval = Date().timeIntervalSince1970) -> TimeInterval {
max(0, ClipboardMaterialFilter.eligibilityDuration - (now - startedAt))
}
}
public enum ClipboardCommandEligibilityTracker: Sendable {
/// Update eligibility from an opportunity-read sample.
/// - Parameters:
/// - changeCount: `UIPasteboard.general.changeCount`
/// - rawText: pasteboard string (may be nil)
/// - previous: last known eligibility
/// - now: clock
public static func refresh(
changeCount: Int,
rawText: String?,
previous: ClipboardCommandEligibility?,
now: TimeInterval = Date().timeIntervalSince1970
) -> ClipboardCommandEligibility? {
guard let rawText else { return nil }
if let previous, previous.changeCount == changeCount {
return previous.isOpen(at: now) ? previous : nil
}
switch ClipboardMaterialFilter.evaluate(rawText) {
case .eligible(let snapshot):
return ClipboardCommandEligibility(
changeCount: changeCount,
snapshot: snapshot,
startedAt: now
)
case .rejected:
return nil
}
}
}
/// In-memory clipboard-command task session (plan §8 layer B). Owned by the keyboard.
public struct ClipboardCommandTaskSession: Equatable, Sendable {
public var snapshot: String
public var previousOutput: String?
public var lastInsertedText: String?
public var expiresAt: TimeInterval
public var fieldFingerprint: String?
public init(
snapshot: String,
previousOutput: String? = nil,
lastInsertedText: String? = nil,
expiresAt: TimeInterval,
fieldFingerprint: String? = nil
) {
self.snapshot = snapshot
self.previousOutput = previousOutput
self.lastInsertedText = lastInsertedText
self.expiresAt = expiresAt
self.fieldFingerprint = fieldFingerprint
}
public func isActive(at now: TimeInterval = Date().timeIntervalSince1970) -> Bool {
now <= expiresAt
}
public mutating func refreshExpiry(at now: TimeInterval = Date().timeIntervalSince1970) {
expiresAt = now + ClipboardMaterialFilter.sessionDuration
}
public mutating func noteSuccessfulInsert(_ text: String, at now: TimeInterval = Date().timeIntervalSince1970) {
lastInsertedText = text
previousOutput = text
refreshExpiry(at: now)
}
}
@@ -0,0 +1,115 @@
// ClipboardCommandPromptComposer.swift
// OSGKeyboard · Shared
//
// Prompt assembly for clipboard-command mode (plan §11).
// Intentionally separate from PolishPromptComposer ASR is an instruction,
// not draft text (R6 must not apply).
import Foundation
public enum ClipboardCommandPromptComposer {
public struct Input: Equatable, Sendable {
public var snapshot: String
public var instruction: String
public var previousOutput: String?
/// Short style bias from the active Style Pack (B1).
public var styleBias: String?
public init(
snapshot: String,
instruction: String,
previousOutput: String? = nil,
styleBias: String? = nil
) {
self.snapshot = snapshot
self.instruction = instruction
self.previousOutput = previousOutput
self.styleBias = styleBias
}
}
public static func compose(_ input: Input, language: AppUILanguage? = nil) -> String {
let useChinese = (language ?? .auto).resolvedLanguageCode().hasPrefix("zh")
var parts: [String] = [useChinese ? chineseCore : englishCore]
if let bias = normalized(input.styleBias), !bias.isEmpty {
let header = useChinese ? "# 语气底色(弱偏置;口述指令优先)" : "# Tone bias (weak; spoken instruction wins)"
parts.append(header)
// Keep bias short so it cannot drown the command contract.
parts.append(String(bias.prefix(800)))
}
return parts.joined(separator: "\n\n")
}
/// User-turn payload (material / instruction / previous output).
public static func userMessage(_ input: Input, language: AppUILanguage? = nil) -> String {
let useChinese = (language ?? .auto).resolvedLanguageCode().hasPrefix("zh")
return userPayload(input, chinese: useChinese)
}
/// B1: derive a short bias string from the active pack without shipping the
/// full dictation personality prompt.
public static func styleBias(
styleID: String,
catalog: PolishStyleCatalog,
maxCharacters: Int = 400
) -> String? {
let pack = PolishStylePackCatalog.resolve(id: styleID, userCatalog: catalog)
let personality = PolishStylePackCatalog.runtimePersonality(for: pack)
let trimmed = personality.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
if trimmed.count <= maxCharacters { return trimmed }
let end = trimmed.index(trimmed.startIndex, offsetBy: maxCharacters)
return String(trimmed[..<end])
}
// MARK: - Private
private static func userPayload(_ input: Input, chinese: Bool) -> String {
var lines: [String] = []
lines.append(chinese ? "【材料】" : "[Material]")
lines.append(ClipboardMaterialFilter.truncateSnapshot(input.snapshot))
lines.append("")
lines.append(chinese ? "【指令】" : "[Instruction]")
lines.append(input.instruction.trimmingCharacters(in: .whitespacesAndNewlines))
if let previous = normalized(input.previousOutput) {
lines.append("")
lines.append(chinese ? "【上一版结果】" : "[Previous output]")
lines.append(previous)
}
return lines.joined(separator: "\n")
}
private static func normalized(_ value: String?) -> String? {
guard let value else { return nil }
let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? nil : trimmed
}
private static let chineseCore = """
你是输入法里的剪贴板写作助手。用户提供一段【材料】(剪贴板内容)和一条【指令】(语音转写)。
你的任务是按指令处理材料,输出用户可以直接发送或粘贴的最终文本。
# 全局契约(最高优先级)
C1 只输出最终文本:不解释、不加引号、不用 markdown 代码块、不写「好的,以下是…」之类前缀。
C2 【指令】优先于任何语气底色;指令要求的语气、目的、篇幅必须遵守。
C3 不要编造材料中没有的关键事实(人名、时间、金额、约定);语气发挥(安慰、拒绝等)允许,但不要捏造情节。
C4 若有【上一版结果】,在上一版基础上按新指令修订,不要重复堆叠无关内容。
C5 材料若注明已截断,只基于可见部分处理。
C6 输出语言跟随指令与材料的主导语言;指令要求翻译时才翻译。
"""
private static let englishCore = """
You are a clipboard writing assistant inside a keyboard. The user provides [Material] (clipboard text) and an [Instruction] (speech transcript).
Produce final text the user can send or paste immediately.
# Global contract (highest priority)
C1 Output final text only: no explanation, quotes, markdown fences, or preamble such as "Sure, here is".
C2 The [Instruction] outranks any tone bias; honor requested tone, intent, and length.
C3 Do not invent key facts absent from the material (names, times, amounts, commitments). Tone (comfort, decline, etc.) may be creative without fabricating plot.
C4 If [Previous output] is present, revise that draft per the new instruction; do not stack unrelated duplicates.
C5 If material is marked truncated, use only the visible portion.
C6 Follow the dominant language of instruction and material; translate only when asked.
"""
}
@@ -0,0 +1,114 @@
// ClipboardMaterialFilter.swift
// OSGKeyboard · Shared
//
// Pure eligibility rules for clipboard-command mode (plan §4 R0R6 content rules).
// Runtime gates (secure field, Full Access) live in the keyboard extension.
import Foundation
public enum ClipboardMaterialFilter: Sendable {
public static let minimumLength = 15
public static let maxSnapshotLength = 3_000
public static let eligibilityDuration: TimeInterval = 30
public static let sessionDuration: TimeInterval = 30
public static let longPressDuration: TimeInterval = 0.45
/// After the host confirms real capture, keep recording at least this long
/// before honoring finger-up (avoids near-silent cold-start tails).
public static let minimumRecordingAfterHostConfirm: TimeInterval = 0.70
public enum Rejection: String, Equatable, Sendable {
case empty
case phoneOrNumeric
case emojiOrSymbolOnly
case verificationCode
case tooShort
case repetitiveSpam
}
public enum Verdict: Equatable, Sendable {
case eligible(String)
case rejected(Rejection)
}
/// Evaluate trimmed clipboard text for command-mode entry.
public static func evaluate(_ raw: String) -> Verdict {
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return .rejected(.empty) }
if isPhoneOrNumeric(trimmed) { return .rejected(.phoneOrNumeric) }
if isEmojiOrSymbolOnly(trimmed) { return .rejected(.emojiOrSymbolOnly) }
if isVerificationCode(trimmed) { return .rejected(.verificationCode) }
if trimmed.count < minimumLength { return .rejected(.tooShort) }
if isRepetitiveSpam(trimmed) { return .rejected(.repetitiveSpam) }
return .eligible(truncateSnapshot(trimmed))
}
/// Wire / LLM snapshot cap (plan: 3000 grapheme clusters).
public static func truncateSnapshot(_ text: String) -> String {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard trimmed.count > maxSnapshotLength else { return trimmed }
let end = trimmed.index(trimmed.startIndex, offsetBy: maxSnapshotLength)
return String(trimmed[..<end])
}
// MARK: - Rules
/// R1: whole string looks like a phone / order number after stripping whitespace.
private static func isPhoneOrNumeric(_ text: String) -> Bool {
let compact = text.filter { !$0.isWhitespace }
guard !compact.isEmpty else { return false }
let allowed = CharacterSet(charactersIn: "0123456789-+()")
guard compact.unicodeScalars.allSatisfy({ allowed.contains($0) }) else { return false }
return compact.contains { $0.isNumber }
}
/// R2: no letter, CJK, or digit only emoji / punctuation / symbols.
private static func isEmojiOrSymbolOnly(_ text: String) -> Bool {
let compact = text.filter { !$0.isWhitespace }
guard !compact.isEmpty else { return false }
return !compact.contains { characterHasLetterOrNumber($0) }
}
/// R3: length 48, alphanumeric only, mixed letters + digits.
private static func isVerificationCode(_ text: String) -> Bool {
let compact = text.filter { !$0.isWhitespace }
guard (4...8).contains(compact.count) else { return false }
guard compact.allSatisfy({ $0.isLetter || $0.isNumber }) else { return false }
let hasLetter = compact.contains(where: \.isLetter)
let hasDigit = compact.contains(where: \.isNumber)
return hasLetter && hasDigit
}
/// R5: length 15, 2 distinct characters, one char 80% share.
private static func isRepetitiveSpam(_ text: String) -> Bool {
let compact = text.filter { !$0.isWhitespace }
guard compact.count >= minimumLength else { return false }
var counts: [Character: Int] = [:]
for ch in compact {
counts[ch, default: 0] += 1
}
guard counts.count <= 2 else { return false }
let maxShare = counts.values.max() ?? 0
return Double(maxShare) / Double(compact.count) >= 0.80
}
private static func characterHasLetterOrNumber(_ character: Character) -> Bool {
if character.isLetter || character.isNumber { return true }
// CJK ideographs / kana counted as letter-like content for R2.
for scalar in character.unicodeScalars {
switch scalar.value {
case 0x4E00...0x9FFF, // CJK Unified
0x3400...0x4DBF, // CJK Ext A
0x3040...0x30FF, // Hiragana / Katakana
0xAC00...0xD7AF: // Hangul
return true
default:
continue
}
}
return false
}
}
@@ -50,8 +50,13 @@ public struct FlowCommand: Codable, Equatable, Sendable {
case startRecording
case stopRecording
case abort
/// Light warm-up: ASR locale/assets only no mic capture.
case prewarm
}
/// Wire version that includes clipboard-command fields.
public static let currentProtocolVersion = 2
public let protocolVersion: Int
public let sessionId: UUID
public let utteranceId: UUID
@@ -60,16 +65,25 @@ public struct FlowCommand: Codable, Equatable, Sendable {
public let localeId: String
public let createdAt: TimeInterval
public let fieldContext: FlowFieldContext?
/// Dictation (default) vs clipboard instruction mode. Absent on legacy v1 dictation.
public let utteranceMode: FlowUtteranceMode?
/// Frozen clipboard material; present on clipboard-command `startRecording`.
public let clipboardSnapshot: String?
/// Prior successful command output for continuous rewrite rounds.
public let previousOutput: String?
public init(
protocolVersion: Int = 1,
protocolVersion: Int = FlowCommand.currentProtocolVersion,
sessionId: UUID,
utteranceId: UUID,
commandSeq: Int64,
action: Action,
localeId: String,
createdAt: TimeInterval = Date().timeIntervalSince1970,
fieldContext: FlowFieldContext? = nil
fieldContext: FlowFieldContext? = nil,
utteranceMode: FlowUtteranceMode? = nil,
clipboardSnapshot: String? = nil,
previousOutput: String? = nil
) {
self.protocolVersion = protocolVersion
self.sessionId = sessionId
@@ -79,6 +93,13 @@ public struct FlowCommand: Codable, Equatable, Sendable {
self.localeId = localeId
self.createdAt = createdAt
self.fieldContext = fieldContext
self.utteranceMode = utteranceMode
self.clipboardSnapshot = clipboardSnapshot
self.previousOutput = previousOutput
}
public var resolvedUtteranceMode: FlowUtteranceMode {
utteranceMode ?? .dictation
}
}
@@ -106,9 +127,11 @@ public struct FlowResult: Codable, Equatable, Sendable {
public let revision: Int64?
public let fieldFingerprint: String?
public let createdAt: TimeInterval
/// Echo of the command mode so the extension can skip raw fallback.
public let utteranceMode: FlowUtteranceMode?
public init(
protocolVersion: Int = 1,
protocolVersion: Int = FlowCommand.currentProtocolVersion,
sessionId: UUID,
utteranceId: UUID,
commandSeq: Int64,
@@ -120,7 +143,8 @@ public struct FlowResult: Codable, Equatable, Sendable {
hostGeneration: String? = nil,
revision: Int64? = nil,
fieldFingerprint: String? = nil,
createdAt: TimeInterval = Date().timeIntervalSince1970
createdAt: TimeInterval = Date().timeIntervalSince1970,
utteranceMode: FlowUtteranceMode? = nil
) {
self.protocolVersion = protocolVersion
self.sessionId = sessionId
@@ -135,6 +159,16 @@ public struct FlowResult: Codable, Equatable, Sendable {
self.revision = revision
self.fieldFingerprint = fieldFingerprint
self.createdAt = createdAt
self.utteranceMode = utteranceMode
}
public var resolvedUtteranceMode: FlowUtteranceMode {
utteranceMode ?? .dictation
}
/// Clipboard-command deliveries must never insert raw ASR into the field.
public var allowsRawFallback: Bool {
resolvedUtteranceMode != .clipboardCommand
}
}
@@ -130,6 +130,10 @@ public final class KeyboardState: ObservableObject {
/// `true` while a cursor-drag pad is being pressed drives the hint
/// shown above the mic.
@Published public var cursorDragActive: Bool = false
/// Opportunity-read: clipboard text is eligible for long-press command mode.
@Published public var clipboardCommandEligible: Bool = false
/// Active clipboard-command task (continuous rewrite window).
@Published public var clipboardCommandSessionActive: Bool = false
/// Whether translate-and-polish is armed for the current engine.
public var isTranslationEffective: Bool {
translationEnabled
@@ -205,6 +209,9 @@ public final class KeyboardState: ObservableObject {
public var beginRecording: () -> Void = {}
public var endRecording: () -> Void = {}
public var tapMic: () -> Void = {}
public var beginClipboardCommand: () -> Void = {}
public var endClipboardCommand: () -> Void = {}
public var refreshClipboardEligibility: () -> Void = {}
public var openSettings: () -> Void = {}
public var startFlowSession: () -> Void = {}
public var setMode: (InputMode) -> Void = { _ in }