rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expand onboarding and adaptive keyboard intelligence

Browse Source 
Add resilient usage analytics, OOBE gateway flows, clipboard semantic ranking, purchase recovery, style learning, and managed current-information search.
This commit is contained in:
Rocky
2026-08-22 16:33:18 +08:00
parent ac374631ae
commit e5a83843db
162 changed files with 23574 additions and 1013 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OSGKeyboardExt/Services/AIKeyboardCoordinator.swift
+81 -7
View File
@@ -18,6 +18,7 @@ final class AIKeyboardCoordinator {
private var requestInsertionFingerprint: String?
private var conversationInsertionFingerprint: String?
private var hasConversationInsertionTarget = false
private var requestOOBEFeature: ManagedGatewayOOBEFeature?
init(
state: KeyboardState,
@@ -37,6 +38,7 @@ final class AIKeyboardCoordinator {
endConversationIfNeeded()
state.aiSession.enter()
requestInsertionFingerprint = nil
requestOOBEFeature = nil
conversationInsertionFingerprint = nil
hasConversationInsertionTarget = false
}
@@ -53,6 +55,7 @@ final class AIKeyboardCoordinator {
endConversationIfNeeded()
state.aiSession.leave()
requestInsertionFingerprint = nil
requestOOBEFeature = nil
conversationInsertionFingerprint = nil
hasConversationInsertionTarget = false
}
@@ -66,7 +69,12 @@ final class AIKeyboardCoordinator {
case .idle, .awaitingSend, .inserted, .sent, .failed:
prepareConversationForRequest()
guard let conversationID = state.aiSession.conversationID else { return }
let disposition = flow.beginAIRecording(conversationID: conversationID)
let oobeFeature = expectedOOBEFeature(.askAI)
requestOOBEFeature = oobeFeature
let disposition = flow.beginAIRecording(
conversationID: conversationID,
oobeFeature: oobeFeature
)
if case .rejected(let rejection) = disposition {
state.aiSession.fail(message(for: rejection), utteranceID: nil)
}
@@ -81,8 +89,23 @@ final class AIKeyboardCoordinator {
/// Tap a clipboard skill chip: same fail-closed material path as hint cards.
func submitClipboardSkill(_ skill: AIClipboardSkill) {
guard canAcceptIdleSubmit else { return }
guard !skill.requiresShortcut
|| state.confirmedClipboardShortcutIDs.contains(skill.id) else {
state.skillTipText = ExtL10n.string("keyboard.ai.skill.shortcutMissing")
return
}
prepareConversationForRequest()
let material = ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
let oobeFeature = oobeFeature(for: skill)
let material: String?
if let oobeFeature {
// OOBE clipboard lessons are intentionally isolated from the
// user's real clipboard history.
material = oobeMaterial(for: oobeFeature)
} else if state.oobePracticeSession != nil {
material = nil
} else {
material = ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
}
if skill.kind == .export {
state.pendingClipboardSkillID = skill.id
state.pendingClipboardSkillSource = material
@@ -113,6 +136,7 @@ final class AIKeyboardCoordinator {
submitResolvedPrompt(
resolution,
taskKind: skill.managedGatewayTaskKind,
oobeFeature: oobeFeature,
thinkingEnabled: skill.thinkingEnabled
)
}
@@ -120,12 +144,19 @@ final class AIKeyboardCoordinator {
/// Tap an idle hint card: resolve its material, skip the mic, ask the host.
func submitHintCard(_ card: AIHintCard) {
guard canAcceptIdleSubmit else { return }
// The OOBE ask-AI lesson must use the explicit hold-to-talk path. Idle
// cards can otherwise pull unrelated clipboard material into a request.
guard state.oobePracticeSession == nil else { return }
prepareConversationForRequest()
let resolution = AIHintPool.resolvePrompt(
for: card,
clipboardText: ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
)
submitResolvedPrompt(resolution, taskKind: .aiQuestion)
submitResolvedPrompt(
resolution,
taskKind: .aiQuestion,
oobeFeature: nil
)
}
private var canAcceptIdleSubmit: Bool {
@@ -141,6 +172,7 @@ final class AIKeyboardCoordinator {
private func submitResolvedPrompt(
_ resolution: AIClipboardPrompt.Resolution,
taskKind: ManagedGatewayTaskKind,
oobeFeature: ManagedGatewayOOBEFeature? = nil,
thinkingEnabled: Bool? = nil
) {
guard case .ready(let prompt) = resolution else {
@@ -156,10 +188,12 @@ final class AIKeyboardCoordinator {
clearPendingExportSkill()
return
}
requestOOBEFeature = oobeFeature
let disposition = flow.submitAIQuestion(
text: prompt,
conversationID: conversationID,
taskKind: taskKind,
oobeFeature: oobeFeature,
thinkingEnabled: thinkingEnabled
)
if case .rejected(let rejection) = disposition {
@@ -172,6 +206,7 @@ final class AIKeyboardCoordinator {
guard state.aiSession.isBusy else { return }
clearPendingExportSkill()
requestInsertionFingerprint = nil
requestOOBEFeature = nil
flow.cancelAIRecording()
state.aiSession.cancelCurrentWork()
}
@@ -179,6 +214,7 @@ final class AIKeyboardCoordinator {
func confirmPendingAnswer() {
if state.aiSession.canInsert, let answer = state.aiSession.answer {
guard insertAnswer(answer) else { return }
markOOBECompletedAfterInsertion()
state.aiSession.markAnswerInserted(
offersSend: state.returnKeyRole.usesActionFill
)
@@ -190,6 +226,7 @@ final class AIKeyboardCoordinator {
func discardPendingAnswer() {
state.aiSession.discardReadyAnswer()
requestInsertionFingerprint = nil
requestOOBEFeature = nil
}
func performCurrentFieldAction() {
@@ -274,6 +311,7 @@ final class AIKeyboardCoordinator {
// fallback when the field or caret changed during generation.
return
}
markOOBECompletedAfterInsertion()
state.aiSession.markAnswerInserted(
offersSend: state.returnKeyRole.usesActionFill
)
@@ -284,6 +322,7 @@ final class AIKeyboardCoordinator {
func fail(_ message: String, utteranceID: UUID?) {
clearPendingExportSkill()
requestInsertionFingerprint = nil
requestOOBEFeature = nil
state.aiSession.fail(message, utteranceID: utteranceID)
}
@@ -319,16 +358,51 @@ final class AIKeyboardCoordinator {
requestInsertionFingerprint = currentFingerprint
}
private func oobeFeature(for skill: AIClipboardSkill) -> ManagedGatewayOOBEFeature? {
switch skill.id {
case AIClipboardSkillCatalog.replyID:
return expectedOOBEFeature(.clipboardReply)
case AIClipboardSkillCatalog.translateID:
return expectedOOBEFeature(.clipboardTranslate)
default:
return nil
}
}
private func expectedOOBEFeature(
_ feature: ManagedGatewayOOBEFeature
) -> ManagedGatewayOOBEFeature? {
guard state.oobePracticeSession?.expectedFeature == feature else { return nil }
return feature
}
private func oobeMaterial(for feature: ManagedGatewayOOBEFeature?) -> String? {
guard let feature,
feature == .clipboardReply || feature == .clipboardTranslate,
let session = state.oobePracticeSession else {
return nil
}
return KeyboardSetupBridge.oobeClipboardMaterial(sessionID: session.sessionID)
}
private func markOOBECompletedAfterInsertion() {
guard let feature = requestOOBEFeature,
let session = state.oobePracticeSession else {
return
}
_ = KeyboardSetupBridge.markOOBEPracticeCompleted(
sessionID: session.sessionID,
feature: feature
)
}
private var isPendingExportSkill: Bool {
guard let id = state.pendingClipboardSkillID else { return false }
return resolvedSkill(id: id)?.kind == .export
}
private func resolvedSkill(id: String) -> AIClipboardSkill? {
AIClipboardSkillCatalog.skill(
id: id,
userCatalog: AppGroupStore().agentUserSkillCatalog
)
state.enabledClipboardSkills.first { $0.id == id }
}
/// Parse an export skill. Empty → in-keyboard tip, stay in the host app.
OSGKeyboardExt/Services/AnalyticsExtensionService.swift
+13
View File
@@ -11,6 +11,7 @@ final class AnalyticsExtensionService: Sendable {
static let shared = AnalyticsExtensionService()
let client: any AnalyticsClient
let keyboardUsageRecorder: any KeyboardUsageRecording
private init() {
let environment = Self.environment
@@ -18,7 +19,15 @@ final class AnalyticsExtensionService: Sendable {
environment: environment,
uploadConfiguration: AnalyticsUploadConfiguration(endpoint: Self.endpoint)
)
let keyboardUsageRuntime = KeyboardUsageRuntime(
environment: environment,
analyticsRepository: runtime.repository,
uploadConfiguration: KeyboardUsageUploadConfiguration(
endpoint: Self.keyboardUsageEndpoint
)
)
client = runtime.client
keyboardUsageRecorder = keyboardUsageRuntime.recorder
}
func recordPresentation(hasFullAccess _: Bool) {
@@ -32,6 +41,10 @@ final class AnalyticsExtensionService: Sendable {
string: "https://account.osglab.com/v1/analytics/events"
)!
private static let keyboardUsageEndpoint = URL(
string: "https://account.osglab.com/v1/analytics/keyboard-usage"
)!
private static var environment: AnalyticsEnvironment {
let appVersion = Bundle.main.object(
forInfoDictionaryKey: "CFBundleShortVersionString"
OSGKeyboardExt/Services/AppGroupPersistor.swift
+21 -2
View File
@@ -42,7 +42,7 @@ public struct AppGroupPersistor {
state.handednessPreference = store.handednessPreference
state.clipboardHistoryEnabled = store.clipboardHistoryEnabled
state.clipboardCandidateBarEnabled = store.clipboardCandidateBarEnabled
state.enabledClipboardSkillIDs = store.agentSkillLayout.enabledIDs
applySkillSnapshot(store: store, into: state)
state.keyboardHapticIntensity = store.keyboardHapticIntensity
applyAPIKeyAvailability(store: store, into: state)
@@ -97,11 +97,30 @@ public struct AppGroupPersistor {
state.handednessPreference = store.handednessPreference
state.clipboardHistoryEnabled = store.clipboardHistoryEnabled
state.clipboardCandidateBarEnabled = store.clipboardCandidateBarEnabled
state.enabledClipboardSkillIDs = store.agentSkillLayout.enabledIDs
applySkillSnapshot(store: store, into: state)
state.keyboardHapticIntensity = store.keyboardHapticIntensity
applyAPIKeyAvailability(store: store, into: state)
}
/// Resolve once outside SwiftUI body evaluation. Assigning the complete
/// value snapshot publishes remote copy changes even when slot IDs are stable.
private func applySkillSnapshot(
store: AppGroupStore,
into state: KeyboardViewController.State
) {
let layout = store.agentSkillLayout
let language = store.uiLanguage
state.uiLanguage = language
state.enabledClipboardSkillIDs = layout.enabledIDs
state.confirmedClipboardShortcutIDs = layout.confirmedShortcutIDs
state.enabledClipboardSkills = AIClipboardSkillCatalog.visible(
enabledIDs: layout.enabledIDs,
officialCatalog: store.officialSkillCatalog,
userCatalog: store.agentUserSkillCatalog,
uiLanguage: language
)
}
/// Cloud without ASR/LLM keys blocks the mic. Local ASR still works when
/// the polish key is missing — show a soft tip above the mic instead.
private func applyAPIKeyAvailability(
OSGKeyboardExt/Services/ClipboardCaptureCoordinator.swift
+21
View File
@@ -34,6 +34,7 @@ final class ClipboardCaptureCoordinator {
private let state: KeyboardState
private let history: ClipboardHistoryStore
private let semanticRanking: ClipboardSemanticRankingStore
private let pasteboard: ClipboardPasteboardProviding
private var pollTimer: Timer?
private var isSecureProvider: () -> Bool = { false }
@@ -48,10 +49,12 @@ final class ClipboardCaptureCoordinator {
init(
state: KeyboardState,
history: ClipboardHistoryStore = .shared,
semanticRanking: ClipboardSemanticRankingStore = .shared,
pasteboard: ClipboardPasteboardProviding = SystemClipboardPasteboard()
) {
self.state = state
self.history = history
self.semanticRanking = semanticRanking
self.pasteboard = pasteboard
}
@@ -79,6 +82,9 @@ final class ClipboardCaptureCoordinator {
// A suggestion belongs to one keyboard presentation. Clear any
// presentation state left behind by a reused extension controller.
endCurrentSuggestion()
if let newest = history.newestAIHintEligibleEntry() {
semanticRanking.analyze(newest)
}
forcesNextSample = true
// Delay the system pasteboard read until the first poll tick. A
// Universal Clipboard fetch or paste alert during the appear sequence
@@ -94,6 +100,7 @@ final class ClipboardCaptureCoordinator {
stopPolling()
// A1 policy: closing the keyboard ends this generation's suggestion.
endCurrentSuggestion()
semanticRanking.clear()
}
func refreshFlagsFromStore() {
@@ -102,6 +109,9 @@ final class ClipboardCaptureCoordinator {
if !state.clipboardHistoryEnabled || !state.clipboardCandidateBarEnabled {
endCurrentSuggestion()
}
if !state.clipboardHistoryEnabled {
semanticRanking.clear()
}
}
func secureEntryDidChange(isSecure: Bool) {
@@ -117,6 +127,7 @@ final class ClipboardCaptureCoordinator {
return
}
endCurrentSuggestion()
semanticRanking.clear()
state.clipboardOverlay = .none
}
@@ -153,12 +164,16 @@ final class ClipboardCaptureCoordinator {
func clearHistory() {
endCurrentSuggestion()
semanticRanking.clear()
history.clearAll()
}
func deleteEntry(id: UUID) {
let deletedChangeCount = history.entries.first(where: { $0.id == id })?.changeCount
history.remove(id: id)
if semanticRanking.snapshot?.entryID == id {
semanticRanking.clear()
}
if deletedChangeCount == state.clipboardSuggestionChangeCount {
endCurrentSuggestion()
}
@@ -233,6 +248,7 @@ final class ClipboardCaptureCoordinator {
// Prefer hasStrings peek before reading body (reduces empty reads).
guard pasteboard.hasStrings else {
semanticRanking.clear()
history.lastObservedChangeCount = changeCount
return
}
@@ -243,7 +259,9 @@ final class ClipboardCaptureCoordinator {
if isCurrentGeneration {
return
}
semanticRanking.clear()
if let entry = history.ingest(rawText: raw, changeCount: changeCount) {
semanticRanking.analyze(entry)
updateSuggestion(with: entry, changeCount: changeCount)
} else {
history.lastObservedChangeCount = changeCount
@@ -306,6 +324,7 @@ final class ClipboardCaptureCoordinator {
// including generations that contain no acceptable text.
clearSuggestion()
guard sample.hasStrings else {
semanticRanking.clear()
history.lastObservedChangeCount = changeCount
return
}
@@ -313,7 +332,9 @@ final class ClipboardCaptureCoordinator {
// republish content from an already observed generation.
guard !isCurrentGeneration else { return }
semanticRanking.clear()
if let entry = history.ingest(rawText: sample.text, changeCount: changeCount) {
semanticRanking.analyze(entry)
updateSuggestion(with: entry, changeCount: changeCount)
} else {
history.lastObservedChangeCount = changeCount
OSGKeyboardExt/Services/KeyboardConfigSync.swift
+15 -1
View File
@@ -92,7 +92,21 @@ final class KeyboardConfigSync {
// Keychain fallback: a reboot must not resurrect the mic gate when
// App Group transiently reads empty.
state.hasCompletedOnboarding = store.hasCompletedOnboarding || Keychain.hasCompletedOnboarding()
state.isOnboardingPracticeActive = KeyboardSetupBridge.isOnboardingPracticeActive
let practice = KeyboardSetupBridge.activeOOBEPracticeSession
state.oobePracticeSession = practice
state.isOnboardingPracticeActive = practice != nil
|| KeyboardSetupBridge.isOnboardingPracticeActive
switch practice?.expectedFeature {
case .voiceInput, .askAI:
state.surface = .voice
case .clipboardTranslate, .clipboardReply:
state.surface = .ai
if !state.aiSession.isActive {
state.aiSession.enter()
}
case nil:
break
}
}
func persistLocale(_ id: String) {
OSGKeyboardExt/Services/KeyboardFlowCoordinator.swift
+20 -3
View File
@@ -677,15 +677,22 @@ final class KeyboardFlowCoordinator {
}
func beginAIRecording(
conversationID: UUID
conversationID: UUID,
oobeFeature: ManagedGatewayOOBEFeature? = nil
) -> FlowUtteranceStartDisposition {
startUtterance(.aiQuestion(conversationID: conversationID))
startUtterance(
.aiQuestion(
conversationID: conversationID,
oobeFeature: oobeFeature
)
)
}
func submitAIQuestion(
text: String,
conversationID: UUID,
taskKind: ManagedGatewayTaskKind = .aiQuestion,
oobeFeature: ManagedGatewayOOBEFeature? = nil,
thinkingEnabled: Bool? = nil
) -> FlowUtteranceStartDisposition {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -695,6 +702,7 @@ final class KeyboardFlowCoordinator {
conversationID: conversationID,
prefilledQuestion: trimmed,
taskKind: taskKind,
oobeFeature: oobeFeature,
thinkingEnabled: thinkingEnabled
)
)
@@ -1181,8 +1189,14 @@ final class KeyboardFlowCoordinator {
let mode: FlowUtteranceMode? = request.mode == .dictation
? nil
: request.mode
let implicitOOBEFeature: ManagedGatewayOOBEFeature? =
state.oobePracticeSession?.expectedFeature == .voiceInput
&& request.mode == .dictation
? .voiceInput
: nil
let managedOOBEFeature = request.managedOOBEFeature ?? implicitOOBEFeature
let managedRequestPurpose = request.managedRequestPurpose
?? (state.isOnboardingPracticeActive && request.mode == .dictation ? .oobe : nil)
?? (managedOOBEFeature == nil ? nil : .oobe)
let command = FlowCommand(
sessionId: activeSessionId,
utteranceId: currentUtteranceId,
@@ -1201,6 +1215,7 @@ final class KeyboardFlowCoordinator {
aiConversationID: request.aiConversationID,
aiTaskKind: request.aiTaskKind,
managedRequestPurpose: managedRequestPurpose,
managedOOBEFeature: managedOOBEFeature,
startDeadlineAt: action == .startRecording ? currentStartDeadlineAt : nil,
processingDeadlineAt: action == .stopRecording && request.isEdit
? Date().timeIntervalSince1970
@@ -1909,6 +1924,8 @@ final class KeyboardFlowCoordinator {
aiConversationID: currentUtteranceRequest?.aiConversationID,
aiQuestionText: text,
aiTaskKind: currentUtteranceRequest?.aiTaskKind,
managedRequestPurpose: currentUtteranceRequest?.managedRequestPurpose,
managedOOBEFeature: currentUtteranceRequest?.managedOOBEFeature,
aiThinkingEnabled: currentUtteranceRequest?.aiThinkingEnabled,
startDeadlineAt: currentStartDeadlineAt
)
OSGKeyboardExt/Services/KeyboardTextInserter.swift
+16 -8
View File
@@ -15,7 +15,7 @@ final class KeyboardTextInserter {
private static let caretVerificationLimit = 80
private let state: KeyboardState
private let insertText: (String) -> Void
private let insertText: (String, KeyboardTextInsertionSource) -> Void
private let deleteBackward: () -> Void
private let contextBeforeInput: () -> String?
private let fieldContextProvider: () -> FlowFieldContext?
@@ -41,7 +41,7 @@ final class KeyboardTextInserter {
init(
state: KeyboardState,
insertText: @escaping (String) -> Void,
insertText: @escaping (String, KeyboardTextInsertionSource) -> Void,
deleteBackward: @escaping () -> Void,
contextBeforeInput: @escaping () -> String?,
fieldContextProvider: @escaping () -> FlowFieldContext?,
@@ -87,8 +87,16 @@ final class KeyboardTextInserter {
insertion: trimmed
)
let inserted = separator + trimmed
insertText(inserted)
insertText(inserted, .voiceTranscription)
KeyboardSetupBridge.markVoiceInsertion()
if delivery.polishWarning == nil,
let practice = state.oobePracticeSession,
practice.expectedFeature == .voiceInput {
_ = KeyboardSetupBridge.markOOBEPracticeCompleted(
sessionID: practice.sessionID,
feature: .voiceInput
)
}
state.noteUserDidInputText()
recordLastInsertion(
inserted,
@@ -120,7 +128,7 @@ final class KeyboardTextInserter {
insertion: trimmed
)
let inserted = separator + trimmed
insertText(inserted)
insertText(inserted, .aiGenerated)
state.noteUserDidInputText()
let mutation = HistoryMutation(
@@ -152,7 +160,7 @@ final class KeyboardTextInserter {
guard !text.isEmpty else { return }
// Verbatim on purpose — paste must reproduce exactly what was copied,
// unlike dictation which needs word-boundary hygiene.
insertText(text)
insertText(text, .pasteboard)
recordUndoableInsertion(text)
// The paste pushed any previous input away from the caret, so the
// "editable last input" hint no longer applies.
@@ -207,7 +215,7 @@ final class KeyboardTextInserter {
state.redoAvailable = false
return
}
insertText(text)
insertText(text, .redo)
recordUndoableInsertion(text)
OSGLog.keyboardExt.info("redo length=\(text.count, privacy: .public)")
}
@@ -375,7 +383,7 @@ final class KeyboardTextInserter {
let inserted = separator + result
transaction.appliedInsertedText = inserted
PendingTextEditTransactionStore.save(transaction)
insertText(inserted)
insertText(inserted, .editGenerated)
let verificationSuffix = String(inserted.suffix(80))
guard contextBeforeInput()?.hasSuffix(verificationSuffix) == true else {
// Never blindly delete after a partial/opaque host insertion. The
@@ -448,7 +456,7 @@ final class KeyboardTextInserter {
switch transaction.deliveryMode {
case .replace:
insertText(transaction.beforeText)
insertText(transaction.beforeText, .editGenerated)
let restore = HistoryMutation(
action: transaction.historyMutation.action == .append ? .delete : .restore,
entryID: transaction.historyMutation.entryID,