feat(keyboard): add clipboard voice command mode
Long-press mic runs ASR as an instruction over eligible clipboard text, with host-confirm recording UI, min-record gate, and light ASR prewarm. Bump CURRENT_PROJECT_VERSION to 53.
This commit is contained in:
@@ -0,0 +1,96 @@
|
||||
// ClipboardCommandEligibility.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Tracks clipboard open-window eligibility (30s from first sighting of a
|
||||
// pasteboard change) for opportunity-read UI. Pure timing logic — no UIKit.
|
||||
|
||||
import Foundation
|
||||
|
||||
public struct ClipboardCommandEligibility: Equatable, Sendable {
|
||||
public var changeCount: Int
|
||||
public var snapshot: String
|
||||
public var startedAt: TimeInterval
|
||||
|
||||
public init(changeCount: Int, snapshot: String, startedAt: TimeInterval = Date().timeIntervalSince1970) {
|
||||
self.changeCount = changeCount
|
||||
self.snapshot = snapshot
|
||||
self.startedAt = startedAt
|
||||
}
|
||||
|
||||
public func isOpen(at now: TimeInterval = Date().timeIntervalSince1970) -> Bool {
|
||||
now - startedAt <= ClipboardMaterialFilter.eligibilityDuration
|
||||
}
|
||||
|
||||
public func remaining(at now: TimeInterval = Date().timeIntervalSince1970) -> TimeInterval {
|
||||
max(0, ClipboardMaterialFilter.eligibilityDuration - (now - startedAt))
|
||||
}
|
||||
}
|
||||
|
||||
public enum ClipboardCommandEligibilityTracker: Sendable {
|
||||
/// Update eligibility from an opportunity-read sample.
|
||||
/// - Parameters:
|
||||
/// - changeCount: `UIPasteboard.general.changeCount`
|
||||
/// - rawText: pasteboard string (may be nil)
|
||||
/// - previous: last known eligibility
|
||||
/// - now: clock
|
||||
public static func refresh(
|
||||
changeCount: Int,
|
||||
rawText: String?,
|
||||
previous: ClipboardCommandEligibility?,
|
||||
now: TimeInterval = Date().timeIntervalSince1970
|
||||
) -> ClipboardCommandEligibility? {
|
||||
guard let rawText else { return nil }
|
||||
|
||||
if let previous, previous.changeCount == changeCount {
|
||||
return previous.isOpen(at: now) ? previous : nil
|
||||
}
|
||||
|
||||
switch ClipboardMaterialFilter.evaluate(rawText) {
|
||||
case .eligible(let snapshot):
|
||||
return ClipboardCommandEligibility(
|
||||
changeCount: changeCount,
|
||||
snapshot: snapshot,
|
||||
startedAt: now
|
||||
)
|
||||
case .rejected:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// In-memory clipboard-command task session (plan §8 layer B). Owned by the keyboard.
|
||||
public struct ClipboardCommandTaskSession: Equatable, Sendable {
|
||||
public var snapshot: String
|
||||
public var previousOutput: String?
|
||||
public var lastInsertedText: String?
|
||||
public var expiresAt: TimeInterval
|
||||
public var fieldFingerprint: String?
|
||||
|
||||
public init(
|
||||
snapshot: String,
|
||||
previousOutput: String? = nil,
|
||||
lastInsertedText: String? = nil,
|
||||
expiresAt: TimeInterval,
|
||||
fieldFingerprint: String? = nil
|
||||
) {
|
||||
self.snapshot = snapshot
|
||||
self.previousOutput = previousOutput
|
||||
self.lastInsertedText = lastInsertedText
|
||||
self.expiresAt = expiresAt
|
||||
self.fieldFingerprint = fieldFingerprint
|
||||
}
|
||||
|
||||
public func isActive(at now: TimeInterval = Date().timeIntervalSince1970) -> Bool {
|
||||
now <= expiresAt
|
||||
}
|
||||
|
||||
public mutating func refreshExpiry(at now: TimeInterval = Date().timeIntervalSince1970) {
|
||||
expiresAt = now + ClipboardMaterialFilter.sessionDuration
|
||||
}
|
||||
|
||||
public mutating func noteSuccessfulInsert(_ text: String, at now: TimeInterval = Date().timeIntervalSince1970) {
|
||||
lastInsertedText = text
|
||||
previousOutput = text
|
||||
refreshExpiry(at: now)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// ClipboardCommandPromptComposer.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Prompt assembly for clipboard-command mode (plan §11).
|
||||
// Intentionally separate from PolishPromptComposer — ASR is an instruction,
|
||||
// not draft text (R6 must not apply).
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum ClipboardCommandPromptComposer {
|
||||
|
||||
public struct Input: Equatable, Sendable {
|
||||
public var snapshot: String
|
||||
public var instruction: String
|
||||
public var previousOutput: String?
|
||||
/// Short style bias from the active Style Pack (B1).
|
||||
public var styleBias: String?
|
||||
|
||||
public init(
|
||||
snapshot: String,
|
||||
instruction: String,
|
||||
previousOutput: String? = nil,
|
||||
styleBias: String? = nil
|
||||
) {
|
||||
self.snapshot = snapshot
|
||||
self.instruction = instruction
|
||||
self.previousOutput = previousOutput
|
||||
self.styleBias = styleBias
|
||||
}
|
||||
}
|
||||
|
||||
public static func compose(_ input: Input, language: AppUILanguage? = nil) -> String {
|
||||
let useChinese = (language ?? .auto).resolvedLanguageCode().hasPrefix("zh")
|
||||
var parts: [String] = [useChinese ? chineseCore : englishCore]
|
||||
|
||||
if let bias = normalized(input.styleBias), !bias.isEmpty {
|
||||
let header = useChinese ? "# 语气底色(弱偏置;口述指令优先)" : "# Tone bias (weak; spoken instruction wins)"
|
||||
parts.append(header)
|
||||
// Keep bias short so it cannot drown the command contract.
|
||||
parts.append(String(bias.prefix(800)))
|
||||
}
|
||||
return parts.joined(separator: "\n\n")
|
||||
}
|
||||
|
||||
/// User-turn payload (material / instruction / previous output).
|
||||
public static func userMessage(_ input: Input, language: AppUILanguage? = nil) -> String {
|
||||
let useChinese = (language ?? .auto).resolvedLanguageCode().hasPrefix("zh")
|
||||
return userPayload(input, chinese: useChinese)
|
||||
}
|
||||
|
||||
/// B1: derive a short bias string from the active pack without shipping the
|
||||
/// full dictation personality prompt.
|
||||
public static func styleBias(
|
||||
styleID: String,
|
||||
catalog: PolishStyleCatalog,
|
||||
maxCharacters: Int = 400
|
||||
) -> String? {
|
||||
let pack = PolishStylePackCatalog.resolve(id: styleID, userCatalog: catalog)
|
||||
let personality = PolishStylePackCatalog.runtimePersonality(for: pack)
|
||||
let trimmed = personality.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return nil }
|
||||
if trimmed.count <= maxCharacters { return trimmed }
|
||||
let end = trimmed.index(trimmed.startIndex, offsetBy: maxCharacters)
|
||||
return String(trimmed[..<end])
|
||||
}
|
||||
|
||||
// MARK: - Private
|
||||
|
||||
private static func userPayload(_ input: Input, chinese: Bool) -> String {
|
||||
var lines: [String] = []
|
||||
lines.append(chinese ? "【材料】" : "[Material]")
|
||||
lines.append(ClipboardMaterialFilter.truncateSnapshot(input.snapshot))
|
||||
lines.append("")
|
||||
lines.append(chinese ? "【指令】" : "[Instruction]")
|
||||
lines.append(input.instruction.trimmingCharacters(in: .whitespacesAndNewlines))
|
||||
if let previous = normalized(input.previousOutput) {
|
||||
lines.append("")
|
||||
lines.append(chinese ? "【上一版结果】" : "[Previous output]")
|
||||
lines.append(previous)
|
||||
}
|
||||
return lines.joined(separator: "\n")
|
||||
}
|
||||
|
||||
private static func normalized(_ value: String?) -> String? {
|
||||
guard let value else { return nil }
|
||||
let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return trimmed.isEmpty ? nil : trimmed
|
||||
}
|
||||
|
||||
private static let chineseCore = """
|
||||
你是输入法里的剪贴板写作助手。用户提供一段【材料】(剪贴板内容)和一条【指令】(语音转写)。
|
||||
你的任务是按指令处理材料,输出用户可以直接发送或粘贴的最终文本。
|
||||
|
||||
# 全局契约(最高优先级)
|
||||
C1 只输出最终文本:不解释、不加引号、不用 markdown 代码块、不写「好的,以下是…」之类前缀。
|
||||
C2 【指令】优先于任何语气底色;指令要求的语气、目的、篇幅必须遵守。
|
||||
C3 不要编造材料中没有的关键事实(人名、时间、金额、约定);语气发挥(安慰、拒绝等)允许,但不要捏造情节。
|
||||
C4 若有【上一版结果】,在上一版基础上按新指令修订,不要重复堆叠无关内容。
|
||||
C5 材料若注明已截断,只基于可见部分处理。
|
||||
C6 输出语言跟随指令与材料的主导语言;指令要求翻译时才翻译。
|
||||
"""
|
||||
|
||||
private static let englishCore = """
|
||||
You are a clipboard writing assistant inside a keyboard. The user provides [Material] (clipboard text) and an [Instruction] (speech transcript).
|
||||
Produce final text the user can send or paste immediately.
|
||||
|
||||
# Global contract (highest priority)
|
||||
C1 Output final text only: no explanation, quotes, markdown fences, or preamble such as "Sure, here is…".
|
||||
C2 The [Instruction] outranks any tone bias; honor requested tone, intent, and length.
|
||||
C3 Do not invent key facts absent from the material (names, times, amounts, commitments). Tone (comfort, decline, etc.) may be creative without fabricating plot.
|
||||
C4 If [Previous output] is present, revise that draft per the new instruction; do not stack unrelated duplicates.
|
||||
C5 If material is marked truncated, use only the visible portion.
|
||||
C6 Follow the dominant language of instruction and material; translate only when asked.
|
||||
"""
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// ClipboardMaterialFilter.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Pure eligibility rules for clipboard-command mode (plan §4 R0–R6 content rules).
|
||||
// Runtime gates (secure field, Full Access) live in the keyboard extension.
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum ClipboardMaterialFilter: Sendable {
|
||||
|
||||
public static let minimumLength = 15
|
||||
public static let maxSnapshotLength = 3_000
|
||||
public static let eligibilityDuration: TimeInterval = 30
|
||||
public static let sessionDuration: TimeInterval = 30
|
||||
public static let longPressDuration: TimeInterval = 0.45
|
||||
/// After the host confirms real capture, keep recording at least this long
|
||||
/// before honoring finger-up (avoids near-silent cold-start tails).
|
||||
public static let minimumRecordingAfterHostConfirm: TimeInterval = 0.70
|
||||
|
||||
public enum Rejection: String, Equatable, Sendable {
|
||||
case empty
|
||||
case phoneOrNumeric
|
||||
case emojiOrSymbolOnly
|
||||
case verificationCode
|
||||
case tooShort
|
||||
case repetitiveSpam
|
||||
}
|
||||
|
||||
public enum Verdict: Equatable, Sendable {
|
||||
case eligible(String)
|
||||
case rejected(Rejection)
|
||||
}
|
||||
|
||||
/// Evaluate trimmed clipboard text for command-mode entry.
|
||||
public static func evaluate(_ raw: String) -> Verdict {
|
||||
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return .rejected(.empty) }
|
||||
|
||||
if isPhoneOrNumeric(trimmed) { return .rejected(.phoneOrNumeric) }
|
||||
if isEmojiOrSymbolOnly(trimmed) { return .rejected(.emojiOrSymbolOnly) }
|
||||
if isVerificationCode(trimmed) { return .rejected(.verificationCode) }
|
||||
if trimmed.count < minimumLength { return .rejected(.tooShort) }
|
||||
if isRepetitiveSpam(trimmed) { return .rejected(.repetitiveSpam) }
|
||||
|
||||
return .eligible(truncateSnapshot(trimmed))
|
||||
}
|
||||
|
||||
/// Wire / LLM snapshot cap (plan: 3000 grapheme clusters).
|
||||
public static func truncateSnapshot(_ text: String) -> String {
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard trimmed.count > maxSnapshotLength else { return trimmed }
|
||||
let end = trimmed.index(trimmed.startIndex, offsetBy: maxSnapshotLength)
|
||||
return String(trimmed[..<end])
|
||||
}
|
||||
|
||||
// MARK: - Rules
|
||||
|
||||
/// R1: whole string looks like a phone / order number after stripping whitespace.
|
||||
private static func isPhoneOrNumeric(_ text: String) -> Bool {
|
||||
let compact = text.filter { !$0.isWhitespace }
|
||||
guard !compact.isEmpty else { return false }
|
||||
let allowed = CharacterSet(charactersIn: "0123456789-+()")
|
||||
guard compact.unicodeScalars.allSatisfy({ allowed.contains($0) }) else { return false }
|
||||
return compact.contains { $0.isNumber }
|
||||
}
|
||||
|
||||
/// R2: no letter, CJK, or digit — only emoji / punctuation / symbols.
|
||||
private static func isEmojiOrSymbolOnly(_ text: String) -> Bool {
|
||||
let compact = text.filter { !$0.isWhitespace }
|
||||
guard !compact.isEmpty else { return false }
|
||||
return !compact.contains { characterHasLetterOrNumber($0) }
|
||||
}
|
||||
|
||||
/// R3: length 4…8, alphanumeric only, mixed letters + digits.
|
||||
private static func isVerificationCode(_ text: String) -> Bool {
|
||||
let compact = text.filter { !$0.isWhitespace }
|
||||
guard (4...8).contains(compact.count) else { return false }
|
||||
guard compact.allSatisfy({ $0.isLetter || $0.isNumber }) else { return false }
|
||||
let hasLetter = compact.contains(where: \.isLetter)
|
||||
let hasDigit = compact.contains(where: \.isNumber)
|
||||
return hasLetter && hasDigit
|
||||
}
|
||||
|
||||
/// R5: length ≥ 15, ≤2 distinct characters, one char ≥ 80% share.
|
||||
private static func isRepetitiveSpam(_ text: String) -> Bool {
|
||||
let compact = text.filter { !$0.isWhitespace }
|
||||
guard compact.count >= minimumLength else { return false }
|
||||
|
||||
var counts: [Character: Int] = [:]
|
||||
for ch in compact {
|
||||
counts[ch, default: 0] += 1
|
||||
}
|
||||
guard counts.count <= 2 else { return false }
|
||||
let maxShare = counts.values.max() ?? 0
|
||||
return Double(maxShare) / Double(compact.count) >= 0.80
|
||||
}
|
||||
|
||||
private static func characterHasLetterOrNumber(_ character: Character) -> Bool {
|
||||
if character.isLetter || character.isNumber { return true }
|
||||
// CJK ideographs / kana counted as “letter-like” content for R2.
|
||||
for scalar in character.unicodeScalars {
|
||||
switch scalar.value {
|
||||
case 0x4E00...0x9FFF, // CJK Unified
|
||||
0x3400...0x4DBF, // CJK Ext A
|
||||
0x3040...0x30FF, // Hiragana / Katakana
|
||||
0xAC00...0xD7AF: // Hangul
|
||||
return true
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -50,8 +50,13 @@ public struct FlowCommand: Codable, Equatable, Sendable {
|
||||
case startRecording
|
||||
case stopRecording
|
||||
case abort
|
||||
/// Light warm-up: ASR locale/assets only — no mic capture.
|
||||
case prewarm
|
||||
}
|
||||
|
||||
/// Wire version that includes clipboard-command fields.
|
||||
public static let currentProtocolVersion = 2
|
||||
|
||||
public let protocolVersion: Int
|
||||
public let sessionId: UUID
|
||||
public let utteranceId: UUID
|
||||
@@ -60,16 +65,25 @@ public struct FlowCommand: Codable, Equatable, Sendable {
|
||||
public let localeId: String
|
||||
public let createdAt: TimeInterval
|
||||
public let fieldContext: FlowFieldContext?
|
||||
/// Dictation (default) vs clipboard instruction mode. Absent on legacy v1 → dictation.
|
||||
public let utteranceMode: FlowUtteranceMode?
|
||||
/// Frozen clipboard material; present on clipboard-command `startRecording`.
|
||||
public let clipboardSnapshot: String?
|
||||
/// Prior successful command output for continuous rewrite rounds.
|
||||
public let previousOutput: String?
|
||||
|
||||
public init(
|
||||
protocolVersion: Int = 1,
|
||||
protocolVersion: Int = FlowCommand.currentProtocolVersion,
|
||||
sessionId: UUID,
|
||||
utteranceId: UUID,
|
||||
commandSeq: Int64,
|
||||
action: Action,
|
||||
localeId: String,
|
||||
createdAt: TimeInterval = Date().timeIntervalSince1970,
|
||||
fieldContext: FlowFieldContext? = nil
|
||||
fieldContext: FlowFieldContext? = nil,
|
||||
utteranceMode: FlowUtteranceMode? = nil,
|
||||
clipboardSnapshot: String? = nil,
|
||||
previousOutput: String? = nil
|
||||
) {
|
||||
self.protocolVersion = protocolVersion
|
||||
self.sessionId = sessionId
|
||||
@@ -79,6 +93,13 @@ public struct FlowCommand: Codable, Equatable, Sendable {
|
||||
self.localeId = localeId
|
||||
self.createdAt = createdAt
|
||||
self.fieldContext = fieldContext
|
||||
self.utteranceMode = utteranceMode
|
||||
self.clipboardSnapshot = clipboardSnapshot
|
||||
self.previousOutput = previousOutput
|
||||
}
|
||||
|
||||
public var resolvedUtteranceMode: FlowUtteranceMode {
|
||||
utteranceMode ?? .dictation
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,9 +127,11 @@ public struct FlowResult: Codable, Equatable, Sendable {
|
||||
public let revision: Int64?
|
||||
public let fieldFingerprint: String?
|
||||
public let createdAt: TimeInterval
|
||||
/// Echo of the command mode so the extension can skip raw fallback.
|
||||
public let utteranceMode: FlowUtteranceMode?
|
||||
|
||||
public init(
|
||||
protocolVersion: Int = 1,
|
||||
protocolVersion: Int = FlowCommand.currentProtocolVersion,
|
||||
sessionId: UUID,
|
||||
utteranceId: UUID,
|
||||
commandSeq: Int64,
|
||||
@@ -120,7 +143,8 @@ public struct FlowResult: Codable, Equatable, Sendable {
|
||||
hostGeneration: String? = nil,
|
||||
revision: Int64? = nil,
|
||||
fieldFingerprint: String? = nil,
|
||||
createdAt: TimeInterval = Date().timeIntervalSince1970
|
||||
createdAt: TimeInterval = Date().timeIntervalSince1970,
|
||||
utteranceMode: FlowUtteranceMode? = nil
|
||||
) {
|
||||
self.protocolVersion = protocolVersion
|
||||
self.sessionId = sessionId
|
||||
@@ -135,6 +159,16 @@ public struct FlowResult: Codable, Equatable, Sendable {
|
||||
self.revision = revision
|
||||
self.fieldFingerprint = fieldFingerprint
|
||||
self.createdAt = createdAt
|
||||
self.utteranceMode = utteranceMode
|
||||
}
|
||||
|
||||
public var resolvedUtteranceMode: FlowUtteranceMode {
|
||||
utteranceMode ?? .dictation
|
||||
}
|
||||
|
||||
/// Clipboard-command deliveries must never insert raw ASR into the field.
|
||||
public var allowsRawFallback: Bool {
|
||||
resolvedUtteranceMode != .clipboardCommand
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -130,6 +130,10 @@ public final class KeyboardState: ObservableObject {
|
||||
/// `true` while a cursor-drag pad is being pressed — drives the hint
|
||||
/// shown above the mic.
|
||||
@Published public var cursorDragActive: Bool = false
|
||||
/// Opportunity-read: clipboard text is eligible for long-press command mode.
|
||||
@Published public var clipboardCommandEligible: Bool = false
|
||||
/// Active clipboard-command task (continuous rewrite window).
|
||||
@Published public var clipboardCommandSessionActive: Bool = false
|
||||
/// Whether translate-and-polish is armed for the current engine.
|
||||
public var isTranslationEffective: Bool {
|
||||
translationEnabled
|
||||
@@ -205,6 +209,9 @@ public final class KeyboardState: ObservableObject {
|
||||
public var beginRecording: () -> Void = {}
|
||||
public var endRecording: () -> Void = {}
|
||||
public var tapMic: () -> Void = {}
|
||||
public var beginClipboardCommand: () -> Void = {}
|
||||
public var endClipboardCommand: () -> Void = {}
|
||||
public var refreshClipboardEligibility: () -> Void = {}
|
||||
public var openSettings: () -> Void = {}
|
||||
public var startFlowSession: () -> Void = {}
|
||||
public var setMode: (InputMode) -> Void = { _ in }
|
||||
|
||||
Reference in New Issue
Block a user