feat(keyboard): ship AI hint carousel, home library cards, and clipboard polish

Rotate AI idle suggestions with optional remote packs, move history/dictionary onto self-sizing Home preview cards, harden clipboard capture/prompting, and simplify keyboard chrome by dropping most liquid-glass shadows.
This commit is contained in:
Rocky
2026-08-13 01:00:51 +08:00
parent fd6e0d3e7e
commit 9f308fadd2
202 changed files with 10897 additions and 5962 deletions
@@ -66,6 +66,38 @@ final class AIKeyboardCoordinator {
}
}
/// Tap an idle hint card: resolve its material, skip the mic, ask the host.
func submitHintCard(_ card: AIHintCard) {
switch state.aiSession.phase {
case .inactive, .idle, .failed:
break
case .preparing, .listening, .recognizing, .generating,
.ready, .awaitingSend, .inserted, .sent:
return
}
enterIfNeeded()
let resolution = AIHintPool.resolvePrompt(
for: card,
clipboardText: ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
)
guard case .ready(let prompt) = resolution else {
// The clipboard window closed between rendering and this tap.
state.aiSession.fail(
ExtL10n.string("keyboard.ai.error.clipboardUnavailable"),
utteranceID: nil
)
return
}
guard let conversationID = state.aiSession.conversationID else { return }
let disposition = flow.submitAIQuestion(
text: prompt,
conversationID: conversationID
)
if case .rejected(let rejection) = disposition {
state.aiSession.fail(message(for: rejection), utteranceID: nil)
}
}
func cancel() {
guard state.aiSession.isBusy else { return }
flow.cancelAIRecording()
+19 -16
View File
@@ -34,7 +34,7 @@ public struct AppGroupPersistor {
// Both engines always polish; ignore legacy off/transcribe modeId.
state.mode = .polish
state.engineMode = store.engineMode
// v0.2.1 follow-up: only the target locale is persisted
// Only the target locale is persisted;
// `translationEnabled` is derived from it. Hydrate once at
// startup; `refreshRuntimeFlags` keeps the chip in sync while
// the keyboard stays open.
@@ -47,23 +47,26 @@ public struct AppGroupPersistor {
applyAPIKeyAvailability(store: store, into: state)
#if DEBUG
// Print a masked view of the live App Group config so we can see
// from the device console exactly what the keyboard extension
// actually sees (and whether it agrees with the main App).
let key = store.apiKey
let masked: String
if key.count > 8 {
masked = "\(key.prefix(4))\(key.suffix(4)) (\(key.count) chars)"
} else if key.isEmpty {
masked = "<empty>"
} else {
masked = "<\(key.count) chars>"
// Log only credential availability and the base URL origin. Never put
// credential fragments or URL path/query/userinfo into device logs.
let credentialStatus: String
switch Keychain.apiKeyOutcome(for: store.providerId, preferICloudSync: true) {
case .found(let value):
credentialStatus = value.isEmpty ? "empty" : "configured"
case .notFound:
credentialStatus = store.apiKey.isEmpty ? "empty" : "configured"
case .unavailable:
credentialStatus = "keychainUnavailable"
}
let components = URLComponents(string: store.baseURL)
let baseURLOrigin = components?.scheme.flatMap { scheme in
components?.host.map { host in "\(scheme)://\(host)" }
} ?? "<invalid>"
print("""
🔍 [AppGroupPersistor.load]
providerId = \(store.providerId)
baseURL = \(store.baseURL)
apiKey = \(masked)
baseURLOrigin = \(baseURLOrigin)
credential = \(credentialStatus)
model = \(store.model)
modeId = \(store.modeId)
localeId = \(store.localeId)
@@ -136,11 +139,11 @@ public struct AppGroupPersistor {
AppGroupStore().setEngineMode(engineMode)
}
/// v0.2.1: persist translation target locale id (e.g. `"en"`,
/// Persist translation target locale id (e.g. `"en"`,
/// `"ja"`, or `TranslationLanguageCatalog.offLocaleId`). The
/// chip / picker call this through `KeyboardState.setTranslationTargetLocaleId`.
///
/// v0.2.1 follow-up: removed `persist(translationEnabled:)` the
/// There is no `persist(translationEnabled:)` because the
/// enabled state is derived from the locale id, so callers only
/// need to write the locale. Keeping the legacy Bool overload
/// around would have implied that there's a separate on/off
@@ -8,20 +8,51 @@ import Foundation
import UIKit
import OSGKeyboardShared
@MainActor
protocol ClipboardPasteboardProviding: AnyObject {
var changeCount: Int { get }
var hasStrings: Bool { get }
var string: String? { get }
}
@MainActor
final class SystemClipboardPasteboard: ClipboardPasteboardProviding {
var changeCount: Int { UIPasteboard.general.changeCount }
var hasStrings: Bool { UIPasteboard.general.hasStrings }
var string: String? { UIPasteboard.general.string }
}
@MainActor
final class ClipboardCaptureCoordinator {
/// Universal Clipboard may synchronously fetch from another device for
/// seconds. System pasteboard reads must never block keyboard presentation.
private static let readQueue = DispatchQueue(
label: "com.osgkeyboard.clipboard.read",
qos: .utility
)
private static let pollInterval: TimeInterval = 0.8
private let state: KeyboardState
private let history: ClipboardHistoryStore
private let pasteboard: ClipboardPasteboardProviding
private var pollTimer: Timer?
private var isSecureProvider: () -> Bool = { false }
private var hasFullAccessProvider: () -> Bool = { false }
/// Ephemeral only: leaving a secure field must not resurrect old body text.
private var secureFieldSuppressedChangeCount: Int?
private var isSecureEntryActive = false
private var isSampling = false
private var forcesNextSample = true
private var isKeyboardVisible = false
init(
state: KeyboardState,
history: ClipboardHistoryStore = .shared
history: ClipboardHistoryStore = .shared,
pasteboard: ClipboardPasteboardProviding = SystemClipboardPasteboard()
) {
self.state = state
self.history = history
self.pasteboard = pasteboard
}
func configure(
@@ -33,22 +64,54 @@ final class ClipboardCaptureCoordinator {
}
func keyboardDidAppear() {
isKeyboardVisible = true
history.reload()
refreshSuggestionFromStore()
captureIfNeeded(force: true)
// A suggestion belongs to one keyboard presentation. Clear any
// presentation state left behind by a reused extension controller.
endCurrentSuggestion()
forcesNextSample = true
// Delay the system pasteboard read until the first poll tick. A
// Universal Clipboard fetch or paste alert during the appear sequence
// can otherwise freeze the keyboard before SwiftUI draws.
if !(pasteboard is SystemClipboardPasteboard) {
captureIfNeeded(forceRead: true)
}
startPolling()
}
func keyboardWillDisappear() {
isKeyboardVisible = false
stopPolling()
// A1 policy: closing the keyboard ends this generation's suggestion.
endCurrentSuggestion()
}
func refreshFlagsFromStore() {
// Called from App Group poll suggestion visibility may change.
refreshSuggestionFromStore()
// Settings changes may hide the active suggestion, but enabling the
// strip must wait for a new pasteboard generation.
if !state.clipboardHistoryEnabled || !state.clipboardCandidateBarEnabled {
endCurrentSuggestion()
}
}
func secureEntryDidChange(isSecure: Bool) {
if isSecure {
isSecureEntryActive = true
secureFieldSuppressedChangeCount = pasteboard.changeCount
} else if isSecureEntryActive {
// Capture the latest generation once more on exit so a pasteboard
// change near the secure-field transition cannot be persisted.
secureFieldSuppressedChangeCount = pasteboard.changeCount
isSecureEntryActive = false
} else {
return
}
endCurrentSuggestion()
state.clipboardOverlay = .none
}
func openPanelFromTopButton() {
guard state.canShowClipboardEntry else { return }
if state.clipboardHistoryEnabled {
history.reload()
state.clipboardOverlay = .historyPanel
@@ -62,15 +125,15 @@ final class ClipboardCaptureCoordinator {
}
func noteUserDidInputText() {
clearSuggestion(persistDismiss: false)
endCurrentSuggestion()
}
func dismissSuggestion() {
history.dismissSuggestion(forChangeCount: state.clipboardSuggestionChangeCount)
clearSuggestion(persistDismiss: true)
endCurrentSuggestion()
}
func insertText(_ text: String, via insert: (String) -> Void) {
guard state.canShowClipboardEntry else { return }
insert(text)
// Tapping a suggestion (or history row that shares this path) must not
// resurface the same clipboard changeCount until the pasteboard changes.
@@ -79,24 +142,28 @@ final class ClipboardCaptureCoordinator {
}
func clearHistory() {
endCurrentSuggestion()
history.clearAll()
clearSuggestion(persistDismiss: false)
}
func deleteEntry(id: UUID) {
let deletedChangeCount = history.entries.first(where: { $0.id == id })?.changeCount
history.remove(id: id)
refreshSuggestionFromStore()
if deletedChangeCount == state.clipboardSuggestionChangeCount {
endCurrentSuggestion()
}
}
// MARK: - Capture
private func startPolling() {
stopPolling()
let timer = Timer(timeInterval: 0.8, repeats: true) { [weak self] _ in
let timer = Timer(timeInterval: Self.pollInterval, repeats: true) { [weak self] _ in
Task { @MainActor in
self?.captureIfNeeded(force: false)
self?.captureIfNeeded()
}
}
timer.tolerance = Self.pollInterval / 4
RunLoop.main.add(timer, forMode: .common)
pollTimer = timer
}
@@ -106,83 +173,170 @@ final class ClipboardCaptureCoordinator {
pollTimer = nil
}
private func captureIfNeeded(force: Bool) {
func captureIfNeeded(forceRead: Bool = false) {
guard state.clipboardHistoryEnabled else {
clearSuggestion(persistDismiss: false)
endCurrentSuggestion()
return
}
#if DEBUG
// What's New demo seeds history itself never touch the pasteboard
// (avoids the simulator alert mid-recording).
if WhatsNewDemoScenario.peek() != nil || WhatsNewDemoScenario.isPlaying() {
return
}
#endif
guard hasFullAccessProvider() else { return }
guard !isSecureProvider() else { return }
let pasteboard = UIPasteboard.general
let changeCount = pasteboard.changeCount
if !force, changeCount == history.lastObservedChangeCount {
refreshSuggestionFromStore()
guard !isSecureProvider() else {
secureEntryDidChange(isSecure: true)
return
}
if pasteboard is SystemClipboardPasteboard {
beginSystemSample(forceRead: forceRead || forcesNextSample)
forcesNextSample = false
return
}
captureInjectedPasteboard(forceRead: forceRead)
}
/// Synchronous path retained for deterministic tests and injected fakes.
/// Production always uses `beginSystemSample` below.
private func captureInjectedPasteboard(forceRead: Bool) {
let changeCount = pasteboard.changeCount
if ClipboardHistoryPolicy.shouldSuppressCapture(
changeCount: changeCount,
secureFieldSuppressedChangeCount: secureFieldSuppressedChangeCount
) {
history.lastObservedChangeCount = changeCount
clearSuggestion()
return
}
secureFieldSuppressedChangeCount = nil
let isCurrentGeneration = changeCount == history.lastObservedChangeCount
if isCurrentGeneration && !forceRead {
return
}
// A new generation replaces any previous transient suggestion,
// including generations that contain no acceptable text.
clearSuggestion()
// Prefer hasStrings peek before reading body (reduces empty reads).
guard pasteboard.hasStrings else {
history.lastObservedChangeCount = changeCount
refreshSuggestionFromStore()
return
}
let raw = pasteboard.string
// The forced appearance read exists only to establish/refresh iOS
// paste permission. It must not reinsert or republish old content.
if isCurrentGeneration {
return
}
if let entry = history.ingest(rawText: raw, changeCount: changeCount) {
updateSuggestion(with: entry, changeCount: changeCount)
} else {
history.lastObservedChangeCount = changeCount
refreshSuggestionFromStore()
}
}
private func refreshSuggestionFromStore() {
guard state.clipboardHistoryEnabled,
state.clipboardCandidateBarEnabled,
!isSecureProvider(),
let newest = history.newestEntry
else {
clearSuggestion(persistDismiss: false)
private struct Sample: Sendable {
let changeCount: Int
let hasStrings: Bool
let text: String?
}
private func beginSystemSample(forceRead: Bool) {
guard !isSampling else { return }
isSampling = true
let lastObserved = history.lastObservedChangeCount
Self.readQueue.async {
let pasteboard = UIPasteboard.general
let changeCount = pasteboard.changeCount
guard forceRead || changeCount != lastObserved else {
Task { @MainActor [weak self] in
self?.isSampling = false
}
return
}
let hasStrings = pasteboard.hasStrings
let sample = Sample(
changeCount: changeCount,
hasStrings: hasStrings,
text: hasStrings ? pasteboard.string : nil
)
Task { @MainActor [weak self] in
self?.finishSystemSample(sample)
}
}
}
private func finishSystemSample(_ sample: Sample) {
isSampling = false
guard isKeyboardVisible,
state.clipboardHistoryEnabled,
hasFullAccessProvider(),
!isSecureProvider()
else { return }
let changeCount = sample.changeCount
if ClipboardHistoryPolicy.shouldSuppressCapture(
changeCount: changeCount,
secureFieldSuppressedChangeCount: secureFieldSuppressedChangeCount
) {
history.lastObservedChangeCount = changeCount
clearSuggestion()
return
}
let changeCount = newest.changeCount ?? history.lastObservedChangeCount
guard history.shouldShowSuggestion(
forChangeCount: changeCount,
candidateBarEnabled: state.clipboardCandidateBarEnabled,
historyEnabled: state.clipboardHistoryEnabled
) else {
clearSuggestion(persistDismiss: false)
secureFieldSuppressedChangeCount = nil
let isCurrentGeneration = changeCount == history.lastObservedChangeCount
// A new generation replaces any previous transient suggestion,
// including generations that contain no acceptable text.
clearSuggestion()
guard sample.hasStrings else {
history.lastObservedChangeCount = changeCount
return
}
// Don't resurrect a strip the user already dismissed this session
// unless changeCount advanced (handled in ingest).
if state.clipboardSuggestionText == nil,
let dismissed = history.suggestionDismissedChangeCount,
dismissed == changeCount {
return
// Forced appearance reads establish iOS paste permission only. Never
// republish content from an already observed generation.
guard !isCurrentGeneration else { return }
if let entry = history.ingest(rawText: sample.text, changeCount: changeCount) {
updateSuggestion(with: entry, changeCount: changeCount)
} else {
history.lastObservedChangeCount = changeCount
}
state.clipboardSuggestionText = newest.text
state.clipboardSuggestionChangeCount = changeCount
}
private func updateSuggestion(with entry: ClipboardHistoryEntry, changeCount: Int) {
guard state.clipboardCandidateBarEnabled else {
clearSuggestion(persistDismiss: false)
guard state.canShowClipboardEntry,
state.clipboardCandidateBarEnabled,
changeCount != secureFieldSuppressedChangeCount
else {
clearSuggestion()
return
}
// Already used/dismissed this pasteboard generation keep it hidden.
if history.suggestionDismissedChangeCount == changeCount {
clearSuggestion(persistDismiss: false)
clearSuggestion()
return
}
state.clipboardSuggestionText = entry.text
state.clipboardSuggestionChangeCount = changeCount
}
private func clearSuggestion(persistDismiss: Bool) {
if persistDismiss {
// already written in dismissSuggestion
private func endCurrentSuggestion() {
history.dismissSuggestion(forChangeCount: state.clipboardSuggestionChangeCount)
clearSuggestion()
}
private func clearSuggestion() {
guard state.clipboardSuggestionText != nil
|| state.clipboardSuggestionChangeCount != nil
else {
return
}
state.clipboardSuggestionText = nil
state.clipboardSuggestionChangeCount = nil
@@ -0,0 +1,71 @@
// EditHintScheduler.swift
// OSGKeyboard · Keyboard Extension
//
// Owns the edit-hint lifetime so every producer shares one expiration order.
import Foundation
import OSGKeyboardShared
@MainActor
final class EditHintScheduler {
typealias Sleeper = @MainActor (Duration) async -> Void
private let state: KeyboardState
private let sleeper: Sleeper
private var task: Task<Void, Never>?
private var generation: UInt64 = 0
init(
state: KeyboardState,
sleeper: @escaping Sleeper = { duration in
try? await Task.sleep(for: duration)
}
) {
self.state = state
self.sleeper = sleeper
}
func show(message: String, isPositive: Bool, duration: Duration) {
let scheduledGeneration = advanceGeneration()
task?.cancel()
state.editHint = message
state.editHintIsPositive = isPositive
let sleeper = sleeper
task = Task { @MainActor [weak self, sleeper] in
await sleeper(duration)
guard let self, self.generation == scheduledGeneration else {
return
}
self.task = nil
self.clearHint()
}
}
func clearPositive() {
guard state.editHintIsPositive else { return }
advanceGeneration()
task?.cancel()
task = nil
clearHint()
}
func invalidate() {
advanceGeneration()
task?.cancel()
task = nil
clearHint()
}
@discardableResult
private func advanceGeneration() -> UInt64 {
generation &+= 1
return generation
}
private func clearHint() {
state.editHint = nil
state.editHintIsPositive = false
}
}
@@ -316,10 +316,6 @@ final class KeyboardFlowCoordinator {
snapshotReason: readySnapshot?.reason
)
state.flowSessionActive = sessionActive
state.debugPendingFlowStart = isPendingFlowStart
state.debugFlowRecording = isFlowRecording
state.debugAwaitingFlowResult = isAwaitingFlowResult
state.debugHasFullAccess = hasFullAccess()
state.micVoiceAvailability = MicVoiceAvailabilityResolver.resolve(
phase: state.phase,
micDisabled: state.micDisabled,
@@ -641,6 +637,17 @@ final class KeyboardFlowCoordinator {
startUtterance(.aiQuestion(conversationID: conversationID))
}
func submitAIQuestion(
text: String,
conversationID: UUID
) -> FlowUtteranceStartDisposition {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return .rejected(.pipelineBusy) }
return startUtterance(
.aiQuestion(conversationID: conversationID, prefilledQuestion: trimmed)
)
}
func stopAIRecording() {
guard currentUtteranceRequest?.isAIQuestion == true else { return }
pressEnded()
@@ -1223,7 +1230,7 @@ final class KeyboardFlowCoordinator {
"status=\(result.status.rawValue) "
+ "kind=\(result.errorKind?.rawValue ?? "none") "
+ "utterance=\(result.utteranceId.uuidString.prefix(8)) "
+ "message=\(result.text ?? "nil")"
+ "messageLen=\(result.text?.count ?? 0)"
)
isAwaitingFlowResult = false
stopFlowWatchdog()
@@ -1738,6 +1745,26 @@ final class KeyboardFlowCoordinator {
}
FlowSessionBridge.setPendingKeyboardUtteranceId(currentUtteranceId)
lastStoppedUtteranceId = nil
// Prefilled AI hint: skip mic / ASR and ask the host to answer text.
if currentUtteranceRequest?.isAIQuestion == true,
let question = currentUtteranceRequest?.aiQuestionText?
.trimmingCharacters(in: .whitespacesAndNewlines),
!question.isEmpty {
writeSubmitAIQuestion(question)
isFlowRecording = false
isAwaitingFlowResult = true
state.lastTranscript = question
state.phase = .processing
if let currentUtteranceId {
onAIRecognitionStarted(currentUtteranceId)
}
startFlowResultWatchdog()
recomputeMicVoiceAvailability()
traceState("startFlowRecording.submitAIQuestion")
return
}
writeCommand(.startRecording)
isFlowRecording = true
state.lastTranscript = ""
@@ -1759,6 +1786,37 @@ final class KeyboardFlowCoordinator {
traceState("startFlowRecording.started")
}
private func writeSubmitAIQuestion(_ text: String) {
guard let activeSessionId, let currentUtteranceId else { return }
let command = FlowCommand(
sessionId: activeSessionId,
utteranceId: currentUtteranceId,
commandSeq: nextCommandSeq(),
action: .submitAIQuestion,
localeId: state.localeId,
utteranceMode: .aiQuestion,
aiConversationID: currentUtteranceRequest?.aiConversationID,
aiQuestionText: text,
startDeadlineAt: currentStartDeadlineAt
)
FlowSessionBridge.writeCommand(command)
if let currentStartDeadlineAt {
FlowSessionBridge.writeStartTransaction(
FlowStartTransaction(
sessionID: activeSessionId,
utteranceID: currentUtteranceId,
deadlineAt: currentStartDeadlineAt,
phase: .issued
)
)
}
FlowTrace.keyboard(
"command.submitAIQuestion",
"seq=\(command.commandSeq) utterance=\(currentUtteranceId.uuidString.prefix(8)) "
+ "chars=\(text.count)"
)
}
private func startUtteranceCountdown() {
utteranceStartedAt = Date().timeIntervalSince1970
state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration)
@@ -2012,7 +2070,7 @@ final class KeyboardFlowCoordinator {
"via=resultWatchdog status=\(result.status.rawValue) "
+ "kind=\(error.kind.rawValue) "
+ "utterance=\(result.utteranceId.uuidString.prefix(8)) "
+ "message=\(error.message)"
+ "messageLen=\(error.message.count)"
)
self.state.phase = .error(
.fromFlowTranscription(error),
@@ -21,6 +21,7 @@ final class KeyboardTextInserter {
private let fieldContextProvider: () -> FlowFieldContext?
private let selectedText: () -> String?
private let scheduleAutoClearError: () -> Void
private unowned let editHintScheduler: EditHintScheduler
/// Exact string last inserted through this inserter dictation, AI answer,
/// edit result or clipboard paste (including any word-boundary separator).
@@ -33,7 +34,6 @@ final class KeyboardTextInserter {
private var redoContextBefore: String?
private let extensionInstanceID = UUID()
private var lastEditUndo: PendingTextEditTransaction?
private var editHintTask: Task<Void, Never>?
/// Suppresses availability refresh while we walk `deleteBackward`
/// for undo, so intermediate contexts don't flicker the button.
private var isUndoing = false
@@ -45,7 +45,8 @@ final class KeyboardTextInserter {
contextBeforeInput: @escaping () -> String?,
fieldContextProvider: @escaping () -> FlowFieldContext?,
selectedText: @escaping () -> String?,
scheduleAutoClearError: @escaping () -> Void
scheduleAutoClearError: @escaping () -> Void,
editHintScheduler: EditHintScheduler
) {
self.state = state
self.insertText = insertText
@@ -54,6 +55,7 @@ final class KeyboardTextInserter {
self.fieldContextProvider = fieldContextProvider
self.selectedText = selectedText
self.scheduleAutoClearError = scheduleAutoClearError
self.editHintScheduler = editHintScheduler
}
func handleFlowTranscript(
@@ -510,28 +512,16 @@ final class KeyboardTextInserter {
extensionInstanceID: extensionInstanceID
)
)
editHintTask?.cancel()
let hint = ExtL10n.string("keyboard.edit.hint.available")
state.editHint = hint
state.editHintIsPositive = true
editHintTask = Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: 10_000_000_000)
guard !Task.isCancelled,
self?.state.editHint == hint,
self?.state.editHintIsPositive == true else {
return
}
self?.state.editHint = nil
self?.state.editHintIsPositive = false
}
editHintScheduler.show(
message: hint,
isPositive: true,
duration: .seconds(10)
)
}
private func clearEditHintIfPositive() {
guard state.editHintIsPositive else { return }
editHintTask?.cancel()
editHintTask = nil
state.editHint = nil
state.editHintIsPositive = false
editHintScheduler.clearPositive()
}
private func clearLastInsertion() {
@@ -15,7 +15,7 @@ final class LastInputEditCoordinator {
private let stopFlow: () -> Void
private let abortFlow: () -> Void
private let acknowledge: (FlowAck.DeliveryOutcome) -> Void
private var hintTask: Task<Void, Never>?
private unowned let editHintScheduler: EditHintScheduler
private var activeUtteranceID: UUID?
private var reviewedUtteranceID: UUID?
private var reviewedRevision: Int64?
@@ -26,7 +26,8 @@ final class LastInputEditCoordinator {
beginFlow: @escaping (EditableInputReference) -> FlowUtteranceStartDisposition,
stopFlow: @escaping () -> Void,
abortFlow: @escaping () -> Void,
acknowledge: @escaping (FlowAck.DeliveryOutcome) -> Void
acknowledge: @escaping (FlowAck.DeliveryOutcome) -> Void,
editHintScheduler: EditHintScheduler
) {
self.state = state
self.textInserter = textInserter
@@ -34,6 +35,7 @@ final class LastInputEditCoordinator {
self.stopFlow = stopFlow
self.abortFlow = abortFlow
self.acknowledge = acknowledge
self.editHintScheduler = editHintScheduler
}
func begin() {
@@ -222,35 +224,14 @@ final class LastInputEditCoordinator {
reviewedRevision = nil
}
func showAvailabilityHintAfterDictation() {
guard textInserter.editableReference() != nil else { return }
showHint(
ExtL10n.string("keyboard.edit.hint.available"),
isPositive: true,
durationNanoseconds: 10_000_000_000
private func showHint(_ message: String) {
editHintScheduler.show(
message: message,
isPositive: false,
duration: .milliseconds(2_500)
)
}
private func showHint(
_ message: String,
isPositive: Bool = false,
durationNanoseconds: UInt64 = 2_500_000_000
) {
hintTask?.cancel()
state.editHint = message
state.editHintIsPositive = isPositive
hintTask = Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: durationNanoseconds)
guard !Task.isCancelled,
self?.state.editHint == message,
self?.state.editHintIsPositive == isPositive else {
return
}
self?.state.editHint = nil
self?.state.editHintIsPositive = false
}
}
private func startFailureMessage(
for disposition: FlowUtteranceStartDisposition
) -> String {
@@ -0,0 +1,317 @@
// WhatsNewDemoDriver.swift
// OSGKeyboard · Keyboard Extension (DEBUG-only)
//
// Plays a scripted What's New timeline on the **real** keyboard surface while
// a Notes-like host sits underneath. Mutates the host document via the
// textDocumentProxy so the clip shows a closed loop (insert / replace).
#if DEBUG
import Foundation
import OSGKeyboardShared
@MainActor
enum WhatsNewDemoDriver {
private static var running = false
private static var activeTask: Task<Void, Never>?
/// Document mutations + Return for AI .
struct HostHooks {
var insertText: (String) -> Void
var deleteBackward: () -> Void
var contextBeforeInput: () -> String?
var performReturn: () -> Void
}
/// Keyboard extensions outlive host relaunches always allow a fresh arm.
static func resetForNewPresentation() {
activeTask?.cancel()
activeTask = nil
running = false
WhatsNewDemoScenario.finishPlaying()
}
static func startIfNeeded(state: KeyboardState, host: HostHooks) {
// Peek first so a brief appear/disappear does not burn the arm.
guard WhatsNewDemoScenario.peek() != nil else { return }
// A still-running timeline from a previous host launch must not block
// the next What's New recording.
if running {
resetForNewPresentation()
}
running = true
activeTask = Task { @MainActor in
// Wait for first layout / surface mount over the host.
try? await Task.sleep(nanoseconds: 900_000_000)
guard !Task.isCancelled else {
running = false
return
}
guard let armed = WhatsNewDemoScenario.consume() else {
running = false
return
}
OSGDiag.log(
"WhatsNewDemo start scenario=\(armed.scenario.rawValue) lang=\(armed.language.rawValue)",
category: "boot"
)
polishDemoChrome(state)
switch armed.scenario {
case .edit:
await runEdit(
state: state,
host: host,
original: armed.seedText,
language: armed.language
)
case .ai:
await runAI(state: state, host: host, language: armed.language)
case .clipboard:
await runClipboard(state: state, host: host, language: armed.language)
}
if !Task.isCancelled {
WhatsNewDemoScenario.finishPlaying()
}
running = false
activeTask = nil
}
}
// MARK: - Edit last input
private static func runEdit(
state: KeyboardState,
host: HostHooks,
original: String,
language: WhatsNewDemoScenario.Language
) async {
let edited = language == .en
? "Hi everyone — we'll hold a planning discussion in Conference Room A at 3pm tomorrow. Please be on time."
: "各位好,明天下午三点在 A 会议室召开方案讨论会,请准时参加。"
let reference = EditableInputReference(
displayText: original,
insertedText: original,
postInsertionFingerprint: nil,
extensionInstanceID: UUID()
)
let source = EditSessionSource(reference: reference)
let review = EditReview(
source: source,
resultText: edited,
utteranceID: UUID()
)
state.surface = .voice
state.editCanReplaceOriginal = true
state.phase = .idle
ClipboardHistoryStore.shared.clearAll()
clearClipboardChrome(state)
polishDemoChrome(state)
// Brief idle so the host note + voice mic are visible together.
try? await sleep(1.0)
state.editSession = .listening(source)
state.lastTranscript = ExtL10n.string("keyboard.edit.status.listening")
state.level = 0.45
for _ in 0..<4 {
try? await sleep(0.28)
state.level = Double.random(in: 0.25...0.85)
polishDemoChrome(state)
}
state.editSession = .processing(source)
state.lastTranscript = ExtL10n.string("keyboard.edit.status.processing")
try? await sleep(1.0)
state.editSession = .review(review)
state.lastTranscript = ExtL10n.string("keyboard.edit.status.review")
try? await sleep(2.4)
state.editSession = .applying(review)
state.lastTranscript = ExtL10n.string("keyboard.edit.status.applying")
// Replace host document so the clip closes the loop.
replaceHostText(from: original, to: edited, host: host)
try? await sleep(1.2)
state.editSession = .inactive
state.editCanReplaceOriginal = false
state.phase = .idle
state.lastTranscript = ""
clearClipboardChrome(state)
polishDemoChrome(state)
try? await sleep(1.2)
}
// MARK: - AI keyboard
private static func runAI(
state: KeyboardState,
host: HostHooks,
language: WhatsNewDemoScenario.Language
) async {
let question = language == .en
? "Where should I go this weekend?"
: "周末去哪儿玩比较合适?"
let answer = language == .en
? "Try a nearby town day trip: morning walk in a park or old street, afternoon café, then a local dinner."
: "可以去近郊走走:上午逛古镇或公园,下午找一家口碑好的咖啡馆休息,傍晚再吃顿当地特色菜。"
state.surface = .ai
state.aiServiceAvailable = true
ClipboardHistoryStore.shared.clearAll()
clearClipboardChrome(state)
polishDemoChrome(state)
// Chat host uses returnKeyType=.send; keep role locked for the clip.
state.returnKeyRole = .send
state.aiSession.enter()
try? await sleep(0.9)
let utteranceID = UUID()
state.aiSession.beginPreparing(utteranceID: utteranceID)
try? await sleep(0.25)
state.aiSession.beginListening(utteranceID: utteranceID)
state.level = 0.4
for _ in 0..<6 {
try? await sleep(0.18)
state.level = Double.random(in: 0.25...0.9)
state.aiSession.updateTranscript(question, utteranceID: utteranceID)
polishDemoChrome(state)
}
state.aiSession.beginRecognizing(utteranceID: utteranceID)
try? await sleep(0.4)
state.aiSession.beginGenerating(question: question, utteranceID: utteranceID)
try? await sleep(0.45)
let chars = Array(answer)
var index = 0
while index < chars.count {
index = min(chars.count, index + 5)
let draft = String(chars.prefix(index))
state.aiSession.receivePartialAnswer(draft, utteranceID: utteranceID)
try? await sleep(0.1)
}
state.aiSession.receiveAnswer(answer, utteranceID: utteranceID)
polishDemoChrome(state)
try? await sleep(1.3)
// Insert on a new line so seed + answer stay readable in the composer.
host.insertText("\n" + answer)
state.aiSession.markAnswerInserted(offersSend: true)
polishDemoChrome(state)
try? await sleep(1.1)
state.aiSession.markAnswerSent()
host.performReturn()
// Stay on AI surface so the beat is not buried by voice chrome.
state.surface = .ai
clearClipboardChrome(state)
polishDemoChrome(state)
try? await sleep(1.4)
}
// MARK: - Clipboard history
private static func runClipboard(
state: KeyboardState,
host: HostHooks,
language: WhatsNewDemoScenario.Language
) async {
let samples = language == .en
? [
"Meeting at 3pm tomorrow",
"Room moved to Building A, 3F",
"Bring the clicker and the deck"
]
: [
"明天下午三点开会",
"会议室改到 A 栋 3 楼",
"请带上投影笔和方案文档"
]
let store = ClipboardHistoryStore.shared
store.clearAll()
for text in samples.reversed() {
_ = store.ingest(rawText: text, changeCount: Int.random(in: 1...9_999))
}
store.reload()
// Persist flags so AppGroupPersistor cannot flip history off mid-demo.
if let defaults = AppGroup.defaultsIfAvailable {
defaults.set(true, forKey: AppGroupConfiguration.Keys.clipboardHistoryEnabled)
defaults.set(true, forKey: AppGroupConfiguration.Keys.clipboardCandidateBarEnabled)
defaults.synchronize()
}
state.surface = .voice
state.clipboardHistoryEnabled = true
state.clipboardCandidateBarEnabled = true
state.phase = .idle
state.clipboardOverlay = .none
polishDemoChrome(state)
// Suggestion strip first (matches the docs copy).
state.clipboardSuggestionText = samples[0]
state.clipboardSuggestionChangeCount = 1
try? await sleep(1.2)
polishDemoChrome(state)
state.clipboardOverlay = .historyPanel
try? await sleep(2.6)
polishDemoChrome(state)
// Tap-to-insert: close panel + write into the host field.
state.clipboardOverlay = .none
host.insertText(samples[0])
polishDemoChrome(state)
try? await sleep(0.9)
// Leave a fresh suggestion strip visible for the next copy cue.
state.clipboardSuggestionText = samples[1]
state.clipboardSuggestionChangeCount = 2
// Hold suggestion with chrome locked clean (no Flow warning flash).
for _ in 0..<8 {
polishDemoChrome(state)
try? await sleep(0.2)
}
}
// MARK: - Host helpers
private static func polishDemoChrome(_ state: KeyboardState) {
state.micDisabledHint = ""
state.micVoiceAvailability = .ready
}
private static func clearClipboardChrome(_ state: KeyboardState) {
state.clipboardOverlay = .none
state.clipboardSuggestionText = nil
state.clipboardSuggestionChangeCount = nil
}
private static func replaceHostText(
from original: String,
to edited: String,
host: HostHooks
) {
// Prefer deleting only the seed suffix so we don't wipe unrelated text.
let before = host.contextBeforeInput() ?? ""
let deleteCount: Int
if before.hasSuffix(original) {
deleteCount = original.count
} else if !before.isEmpty {
deleteCount = before.count
} else {
deleteCount = original.count
}
for _ in 0..<deleteCount {
host.deleteBackward()
}
host.insertText(edited)
}
/// ~1.6× slow-mo for screen recording readability (30 fps source).
private static func sleep(_ seconds: Double) async throws {
try await Task.sleep(nanoseconds: UInt64(seconds * 1.6 * 1_000_000_000))
}
}
#endif