Rocky
e5a83843db
Add resilient usage analytics, OOBE gateway flows, clipboard semantic ranking, purchase recovery, style learning, and managed current-information search.
473 lines
21 KiB
Swift
473 lines
21 KiB
Swift
// KeyboardState.swift
|
|
// OSGKeyboard · Shared
|
|
//
|
|
// View-model for the keyboard extension. Lives in Shared (not the
|
|
// extension target) so unit tests can import it directly without the
|
|
// `app-extension` linking headaches. The keyboard view controller
|
|
// (`KeyboardViewController`) re-exports the same type as a typealias so
|
|
// existing call sites (`KeyboardViewController.State`) keep compiling.
|
|
|
|
import Combine
|
|
import Foundation
|
|
import SwiftUI
|
|
import UIKit
|
|
|
|
@MainActor
|
|
public final class KeyboardState: ObservableObject {
|
|
public init() {}
|
|
|
|
/// Full-height clipboard UI layered over the active keyboard surface.
|
|
public enum ClipboardKeyboardOverlay: Equatable {
|
|
case none
|
|
case enableGuide
|
|
case historyPanel
|
|
}
|
|
|
|
/// Pipeline phase. Errors are structured so the UI layer can choose
|
|
/// the right icon / copy for each failure mode without
|
|
/// reverse-parsing a free-form string.
|
|
public enum Phase: Equatable {
|
|
case idle
|
|
case requestingPermissions
|
|
case recording
|
|
case processing
|
|
case error(ErrorKind, message: String? = nil)
|
|
case denied(Reason)
|
|
|
|
/// Why the pipeline failed. `message` is a short, user-facing
|
|
/// hint (e.g. "请检查主 App 设置"); the structured kind is what
|
|
/// drives icon / colour.
|
|
public enum ErrorKind: Equatable {
|
|
case micDenied
|
|
case speechDenied
|
|
case asr(String)
|
|
case llm(LLMError)
|
|
case appGroupUnavailable
|
|
/// Keyboard extension lacks Full Access for host-app jumps.
|
|
case fullAccessRequired
|
|
/// Auto-jump to the host app failed; user must open it manually.
|
|
case manualOpenRequired
|
|
/// Host delivered raw transcript; polish step failed or was skipped.
|
|
case polishDegraded(String)
|
|
/// Host ASR finished with no usable speech.
|
|
case noSpeechDetected
|
|
/// Host ASR was interrupted before a final transcript arrived.
|
|
case recognitionInterrupted
|
|
/// Host could not start background audio capture.
|
|
case hostAudioUnavailable
|
|
/// Host ASR or pipeline failed with a user-facing message.
|
|
case hostTranscriptionFailed(String)
|
|
/// Flow result did not arrive before the keyboard watchdog expired.
|
|
case flowResultTimeout
|
|
/// Host Flow session ended while the keyboard was idle.
|
|
case flowSessionExpired
|
|
case unknown(String)
|
|
}
|
|
|
|
public enum Reason: Equatable { case mic, speech }
|
|
}
|
|
|
|
/// Voice input always runs through polish; legacy off/transcribe modes removed.
|
|
public enum InputMode: String, CaseIterable, Identifiable {
|
|
case polish
|
|
|
|
public var id: String { rawValue }
|
|
|
|
public var labelKey: String { "mode.polish" }
|
|
}
|
|
|
|
/// Keyboard chrome surface. Voice is the default product mode; typing is
|
|
/// a secondary QWERTY / pinyin surface for quick corrections.
|
|
public enum Surface: String, CaseIterable, Identifiable, Sendable {
|
|
case voice
|
|
case typing
|
|
case ai
|
|
|
|
public var id: String { rawValue }
|
|
}
|
|
|
|
@Published public var phase: Phase = .idle
|
|
/// Active chrome. Forced to `.voice` while recording / processing.
|
|
@Published public var surface: Surface = .voice
|
|
@Published public var level: Double = 0
|
|
@Published public var mode: InputMode = .polish
|
|
@Published public var localeId: String = "auto"
|
|
@Published public var lastTranscript: String = ""
|
|
/// `true` if the active ASR session is running on-device for the
|
|
/// current locale. With iOS 26's `SpeechAnalyzer` this is always
|
|
/// `true` — kept on the state object because the UI's status
|
|
/// badge still wants a single source of truth to read from.
|
|
@Published public var onDeviceSupported: Bool = false
|
|
/// Seconds remaining in the current utterance (Flow tap-to-talk).
|
|
@Published public var utteranceRemainingSeconds: Int = Int(FlowSessionKeys.maxUtteranceDuration)
|
|
/// Whether the host app's Flow voice session is live and reachable (fresh
|
|
/// heartbeat). Do not use the App Group session flag alone for UI gating.
|
|
/// Prefer `micVoiceAvailability` for mic color and tap behavior.
|
|
@Published public var flowSessionActive: Bool = false
|
|
/// Unified mic color / tap / hint source for the keyboard extension.
|
|
@Published public var micVoiceAvailability: MicVoiceAvailability = .unavailable(.hostNotReady)
|
|
/// Short-lived host-owned practice mode. It unlocks real dictation before
|
|
/// onboarding completion, but only while the onboarding text field is live.
|
|
@Published public var isOnboardingPracticeActive: Bool = false
|
|
/// Current strict OOBE contract used to render feature-specific extension
|
|
/// state and bind completion to the host-owned session ID.
|
|
@Published public var oobePracticeSession: OOBEPracticeSession?
|
|
/// When true, the mic is intentionally disabled (e.g. cloud engine
|
|
/// selected but the provider-specific API key is missing).
|
|
@Published public var micDisabled: Bool = false
|
|
/// AI mode always needs an LLM even when local ASR keeps voice dictation usable.
|
|
@Published public var aiServiceAvailable: Bool = true
|
|
/// One-line helper shown above the mic while `micDisabled == true`.
|
|
@Published public var micDisabledHint: String = ""
|
|
/// "local" → on-device ASR only. "cloud" → cloud ASR + LLM polish.
|
|
/// Boot value must match the privacy-safe app default (`local`) so the
|
|
/// keyboard never assumes the audio-uploading engine before the App
|
|
/// Group config has been read.
|
|
@Published public var engineMode: String = "local"
|
|
/// Derived: translation is on iff a target
|
|
/// locale has been selected (mirrors `ProviderConfig.translationEnabled`
|
|
/// so the chip / pipeline read the same source of truth).
|
|
public var translationEnabled: Bool {
|
|
translationTargetLocaleId != TranslationLanguageCatalog.offLocaleId
|
|
}
|
|
/// Target locale id the translate-and-polish prompt should
|
|
/// produce (e.g. `"en"`, `"ja"`). Mirrored from `ProviderConfig`.
|
|
/// Defaults to `offLocaleId` so the keyboard boots in the "off"
|
|
/// state on first install.
|
|
@Published public var translationTargetLocaleId: String = TranslationLanguageCatalog.offLocaleId
|
|
/// Mirrored from App Group — swaps delete / space on the bottom row.
|
|
@Published public var handednessPreference: HandednessPreference = .left
|
|
/// Mirrors the host field's return-key intent. The action stays a newline
|
|
/// insert; host apps decide whether that submits or creates a line break.
|
|
@Published public var returnKeyRole: ReturnKeyRole = .newline
|
|
/// Opt-in clipboard history capture (mirrored from App Group).
|
|
@Published public var clipboardHistoryEnabled: Bool = false
|
|
/// Opt-in clipboard suggestion strip (requires history enabled).
|
|
@Published public var clipboardCandidateBarEnabled: Bool = false
|
|
/// Skills-tab order for clipboard chips. Empty → hint carousel.
|
|
@Published public var enabledClipboardSkillIDs: [String] = AIAgentSkillLayout.defaultEnabledIDs
|
|
/// Fully resolved enabled skills. Mirroring value-semantic content here
|
|
/// ensures Darwin updates publish prompt/name changes even when IDs stay unchanged.
|
|
@Published public var enabledClipboardSkills: [AIClipboardSkill] =
|
|
AIClipboardSkillCatalog.visible()
|
|
/// Export skills whose companion Shortcut setup the user confirmed.
|
|
@Published public var confirmedClipboardShortcutIDs: [String] = []
|
|
/// App language captured with the same App Group snapshot as skill copy.
|
|
@Published public var uiLanguage: AppUILanguage = .auto
|
|
/// Export skill currently waiting on the LLM. Nil for transform skills.
|
|
@Published public var pendingClipboardSkillID: String?
|
|
/// Clipboard captured when that export skill was tapped, so the body
|
|
/// still exists after the 30-second hint window closes.
|
|
public var pendingClipboardSkillSource: String?
|
|
/// In-keyboard toast (e.g. no todos). Does not leave the host app.
|
|
@Published public var skillTipText: String?
|
|
/// Host field is a password / secure entry — never read pasteboard.
|
|
@Published public var isSecureTextEntry: Bool = false
|
|
/// Secure fields hide every clipboard-history entry point.
|
|
public var canShowClipboardEntry: Bool {
|
|
!isSecureTextEntry
|
|
}
|
|
/// Full-keyboard clipboard overlay (enable guide or history list).
|
|
@Published public var clipboardOverlay: ClipboardKeyboardOverlay = .none
|
|
/// Suggestion strip above keys (newest clipboard item).
|
|
@Published public var clipboardSuggestionText: String?
|
|
/// Pasteboard changeCount associated with the current suggestion (for dismiss).
|
|
@Published public var clipboardSuggestionChangeCount: Int?
|
|
/// Typing-grid haptic strength (off / light / strong).
|
|
@Published public var keyboardHapticIntensity: KeyboardHapticIntensity = .default
|
|
/// Single source of truth for selecting iPad-scale keyboard metrics.
|
|
/// The view controller resolves this from device idiom + horizontal size
|
|
/// class so SwiftUI and the UIKit height constraint cannot disagree.
|
|
@Published public var usesIPadLayoutMetrics: Bool = false
|
|
/// The custom system-keyboard switch is iPad-only. iPhone relies on the
|
|
/// system-provided switch below the keyboard instead of showing a duplicate.
|
|
@Published public var showsSystemGlobeKey: Bool = false
|
|
/// Width the controller sized the keyboard to. Both the UIKit height
|
|
/// constraint and the SwiftUI key grid pick their metrics from this one
|
|
/// value so they can never disagree and clip the bottom row.
|
|
@Published public var layoutWidth: CGFloat = 0
|
|
/// `true` when the last voice insertion is still at the caret and can
|
|
/// be undone (suffix-checked against `documentContextBeforeInput`).
|
|
@Published public var undoAvailable: Bool = false
|
|
/// A verified OSG-generated insertion can be edited by voice.
|
|
@Published public var editAvailable: Bool = false
|
|
/// `true` while an undone voice insertion can be re-applied (redo buffer).
|
|
@Published public var redoAvailable: Bool = false
|
|
/// `true` when the host field has a non-empty selection (copy enabled).
|
|
@Published public var copyAvailable: Bool = false
|
|
/// `true` when the host field has a non-empty selection (cut enabled).
|
|
@Published public var cutAvailable: Bool = false
|
|
/// Closed state machine for long-press editing of the last insertion.
|
|
@Published public var editSession: EditSessionState = .inactive
|
|
/// AI conversation UI state for the keyboard surface. The host owns the actual messages.
|
|
@Published public var aiSession: AISessionState = .inactive
|
|
/// Whether the assistant's field action can invoke the host Return key.
|
|
/// Derived from the focused field's role and `UITextDocumentProxy.hasText`.
|
|
@Published public var assistantActionAvailable: Bool = false
|
|
/// Brief success pulse rendered on the unified assistant microphone.
|
|
@Published public var assistantInsertionSucceeded: Bool = false
|
|
@Published public var editCanReplaceOriginal: Bool = false
|
|
/// Short idle feedback (availability, expiry, missing LLM).
|
|
@Published public var editHint: String?
|
|
/// Availability hints use the green accent; failures keep warning styling.
|
|
@Published public var editHintIsPositive: Bool = false
|
|
/// Whether translate-and-polish is armed for the current engine.
|
|
public var isTranslationEffective: Bool {
|
|
translationEnabled
|
|
}
|
|
|
|
/// Convenience shorthand used by the pipeline and views.
|
|
public var isLocalEngine: Bool { engineMode == "local" }
|
|
|
|
/// Applies the non-persistent secure-field UI policy immediately.
|
|
public func setSecureTextEntry(_ isSecure: Bool) {
|
|
isSecureTextEntry = isSecure
|
|
guard isSecure else { return }
|
|
clipboardSuggestionText = nil
|
|
clipboardSuggestionChangeCount = nil
|
|
clipboardOverlay = .none
|
|
}
|
|
|
|
// MARK: - Host-app onboarding gate
|
|
|
|
/// Mirrored from App Group / Keychain. Setup UI lives only in the host
|
|
/// app; the keyboard uses this flag to gate voice (mic / Flow cold-start)
|
|
/// and prompt a jump back to the app when incomplete.
|
|
@Published public var hasCompletedOnboarding: Bool = false
|
|
|
|
public enum ReturnKeyRole: Equatable {
|
|
case newline
|
|
case send
|
|
case go
|
|
case search
|
|
case join
|
|
case done
|
|
case next
|
|
case `continue`
|
|
case route
|
|
case google
|
|
case yahoo
|
|
case emergencyCall
|
|
|
|
public var titleKey: String {
|
|
switch self {
|
|
case .newline: return "common.newline"
|
|
case .send: return "common.send"
|
|
case .go: return "keyboard.return.go"
|
|
case .search: return "keyboard.return.search"
|
|
case .join: return "keyboard.return.join"
|
|
case .done: return "common.done"
|
|
case .next: return "keyboard.return.next"
|
|
case .continue: return "common.continue"
|
|
case .route: return "keyboard.return.route"
|
|
case .google: return "keyboard.return.google"
|
|
case .yahoo: return "keyboard.return.yahoo"
|
|
case .emergencyCall: return "keyboard.return.emergencyCall"
|
|
}
|
|
}
|
|
|
|
/// Green action chrome (system uses blue for Go / Search / Send / Done).
|
|
public var usesActionFill: Bool {
|
|
self != .newline
|
|
}
|
|
|
|
/// SF Symbol matching the semantic Return action exposed by the host.
|
|
public var assistantActionSystemImage: String {
|
|
switch self {
|
|
case .newline:
|
|
return "arrow.turn.down.left"
|
|
case .send:
|
|
return "paperplane.fill"
|
|
case .search, .google, .yahoo:
|
|
return "magnifyingglass"
|
|
case .go, .route:
|
|
return "arrow.right.circle.fill"
|
|
case .join:
|
|
return "person.badge.plus"
|
|
case .done:
|
|
return "checkmark"
|
|
case .next, .continue:
|
|
return "arrow.right"
|
|
case .emergencyCall:
|
|
return "phone.fill"
|
|
}
|
|
}
|
|
|
|
/// Stable suffix for UI automation and diagnostics.
|
|
public var assistantActionIdentifier: String {
|
|
switch self {
|
|
case .newline: return "newline"
|
|
case .send: return "send"
|
|
case .go: return "go"
|
|
case .search: return "search"
|
|
case .join: return "join"
|
|
case .done: return "done"
|
|
case .next: return "next"
|
|
case .continue: return "continue"
|
|
case .route: return "route"
|
|
case .google: return "google"
|
|
case .yahoo: return "yahoo"
|
|
case .emergencyCall: return "emergencyCall"
|
|
}
|
|
}
|
|
|
|
/// Text-producing actions need content; navigation actions and newline
|
|
/// remain useful in an empty field.
|
|
public func assistantActionAvailable(hasText: Bool) -> Bool {
|
|
switch self {
|
|
case .send, .go, .search, .join, .route, .google, .yahoo:
|
|
return hasText
|
|
case .newline, .done, .next, .continue, .emergencyCall:
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
// Action hooks — injected by the view controller at install time.
|
|
public var beginRecording: () -> Void = {}
|
|
public var endRecording: () -> Void = {}
|
|
public var tapMic: () -> Void = {}
|
|
/// Starts/cancels a bounded host-audio prime from the user's mic touch.
|
|
public var setMicTouchActive: (Bool) -> Void = { _ in }
|
|
/// Discards the complete normal-dictation round, including late ASR/LLM output.
|
|
public var cancelVoiceInput: () -> Void = {}
|
|
public var beginEditLastInput: () -> Void = {}
|
|
public var stopEditListening: () -> Void = {}
|
|
public var confirmEditResult: () -> Void = {}
|
|
public var closeEditMode: () -> Void = {}
|
|
public var tapAIMic: () -> Void = {}
|
|
public var cancelAIInput: () -> Void = {}
|
|
/// Explicitly inserts a retained AI result after target validation failed.
|
|
public var confirmPendingAIAnswer: () -> Void = {}
|
|
public var discardPendingAIAnswer: () -> Void = {}
|
|
/// Performs the focused field's semantic Return action.
|
|
public var performAssistantFieldAction: () -> Void = {}
|
|
/// Sends a tapped idle hint card as the AI question (skip microphone).
|
|
public var submitAIHint: (AIHintCard) -> Void = { _ in }
|
|
/// Sends a clipboard skill (reply / summarize / translate / export).
|
|
public var submitAIClipboardSkill: (AIClipboardSkill) -> Void = { _ in }
|
|
/// Writes extract-todos titles and opens the host to run the Shortcut.
|
|
public var runClipboardExportSkill: (String, [String]) -> Void = { _, _ in }
|
|
public var openSettings: () -> Void = {}
|
|
/// Opens the host app straight to input-resource deployment. Used by the
|
|
/// typing surface when Rime resources have not been deployed yet.
|
|
public var openInputMethodSetup: () -> Void = {}
|
|
/// Opens the host app Settings → Clipboard page (enable history toggle).
|
|
public var openClipboardSettings: () -> Void = {}
|
|
/// Top-bar clipboard button: guide when history off, else history panel.
|
|
public var openClipboardPanel: () -> Void = {}
|
|
public var dismissClipboardOverlay: () -> Void = {}
|
|
public var insertClipboardText: (String) -> Void = { _ in }
|
|
public var dismissClipboardSuggestion: () -> Void = {}
|
|
public var clearClipboardHistory: () -> Void = {}
|
|
public var deleteClipboardHistoryEntry: (UUID) -> Void = { _ in }
|
|
/// Notify that the user inserted text (hides suggestion strip).
|
|
public var noteUserDidInputText: () -> Void = {}
|
|
/// System globe (🌐) key target. Kept weak to avoid a state → controller
|
|
/// ownership cycle; UIKit's standard all-touch-events action provides both
|
|
/// tap-to-advance and long-press input-mode selection.
|
|
public weak var inputModeController: UIInputViewController?
|
|
public var startFlowSession: () -> Void = {}
|
|
public var setMode: (InputMode) -> Void = { _ in }
|
|
public var setLocale: (String) -> Void = { _ in }
|
|
public var setEngineMode: (String) -> Void = { _ in }
|
|
/// Only the locale picker remains; `enabled`
|
|
/// is derived from the locale id, so there's no separate toggle to
|
|
/// persist. Wired in `KeyboardViewController.installStateActions`.
|
|
public var setTranslationTargetLocaleId: (String) -> Void = { _ in }
|
|
public var insertNewline: () -> Void = {}
|
|
public var insertSpace: () -> Void = {}
|
|
public var deleteBackward: () -> Void = {}
|
|
/// Undo the last voice insertion when `undoAvailable` is true.
|
|
public var undoLastInsertion: () -> Void = {}
|
|
/// Redo the last undone voice insertion when `redoAvailable` is true.
|
|
public var redoLastInsertion: () -> Void = {}
|
|
/// Copy the current text selection to the pasteboard.
|
|
public var copySelection: () -> Void = {}
|
|
/// Cut the current text selection (copy + delete).
|
|
public var cutSelection: () -> Void = {}
|
|
/// Switch voice ↔ typing. No-ops when voice pipeline is active.
|
|
public var setSurface: (Surface) -> Void = { _ in }
|
|
|
|
/// Recording / processing must stay on the voice surface.
|
|
public var locksTypingSurface: Bool {
|
|
if editSession.isActive { return true }
|
|
if aiSession.isBusy { return true }
|
|
switch phase {
|
|
case .requestingPermissions, .recording, .processing:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
public var canEnterTypingSurface: Bool { !locksTypingSurface }
|
|
|
|
public var canCancelAIInput: Bool {
|
|
aiSession.isBusy
|
|
}
|
|
|
|
/// Normal dictation can be discarded from microphone startup through
|
|
/// ASR / polish, including the abort-wait after Cancel until the host
|
|
/// acks (coordinator keeps `phase == .processing` for that window).
|
|
public var canCancelVoiceInput: Bool {
|
|
guard !editSession.isActive else { return false }
|
|
switch phase {
|
|
case .requestingPermissions, .recording, .processing:
|
|
return true
|
|
case .idle, .error, .denied:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// MARK: - Preview helpers (DEBUG only)
|
|
|
|
#if DEBUG
|
|
public static var previewIdle: KeyboardState {
|
|
let s = KeyboardState()
|
|
s.phase = .idle
|
|
s.level = 0
|
|
s.mode = .polish
|
|
s.localeId = "zh-Hans"
|
|
s.lastTranscript = ""
|
|
return s
|
|
}
|
|
public static var previewRecording: KeyboardState {
|
|
let s = KeyboardState()
|
|
s.phase = .recording
|
|
s.level = 0.65
|
|
s.mode = .polish
|
|
s.localeId = "zh-Hans"
|
|
s.lastTranscript = "你好,我想说一段测试"
|
|
return s
|
|
}
|
|
public static var previewProcessing: KeyboardState {
|
|
let s = KeyboardState()
|
|
s.phase = .processing
|
|
s.level = 0
|
|
s.mode = .polish
|
|
s.localeId = "zh-Hans"
|
|
s.lastTranscript = ""
|
|
return s
|
|
}
|
|
#endif
|
|
}
|
|
|
|
extension KeyboardState.Phase.ErrorKind {
|
|
/// Maps a host-app Flow transcription failure into a keyboard error kind.
|
|
public static func fromFlowTranscription(_ error: FlowTranscriptionError) -> Self {
|
|
switch error.kind {
|
|
case .noSpeech:
|
|
return .noSpeechDetected
|
|
case .recognitionInterrupted:
|
|
return .recognitionInterrupted
|
|
case .audioUnavailable:
|
|
return .hostAudioUnavailable
|
|
case .asrFailed, .generic:
|
|
return .hostTranscriptionFailed(error.message)
|
|
case .discardedEmpty:
|
|
return .noSpeechDetected
|
|
}
|
|
}
|
|
}
|