feat(keyboard): ship AI hint carousel, home library cards, and clipboard polish
Rotate AI idle suggestions with optional remote packs, move history/dictionary onto self-sizing Home preview cards, harden clipboard capture/prompting, and simplify keyboard chrome by dropping most liquid-glass shadows.
This commit is contained in:
@@ -66,6 +66,38 @@ final class AIKeyboardCoordinator {
|
||||
}
|
||||
}
|
||||
|
||||
/// Tap an idle hint card: resolve its material, skip the mic, ask the host.
|
||||
func submitHintCard(_ card: AIHintCard) {
|
||||
switch state.aiSession.phase {
|
||||
case .inactive, .idle, .failed:
|
||||
break
|
||||
case .preparing, .listening, .recognizing, .generating,
|
||||
.ready, .awaitingSend, .inserted, .sent:
|
||||
return
|
||||
}
|
||||
enterIfNeeded()
|
||||
let resolution = AIHintPool.resolvePrompt(
|
||||
for: card,
|
||||
clipboardText: ClipboardHistoryStore.shared.newestAIHintEligibleEntry()?.text
|
||||
)
|
||||
guard case .ready(let prompt) = resolution else {
|
||||
// The clipboard window closed between rendering and this tap.
|
||||
state.aiSession.fail(
|
||||
ExtL10n.string("keyboard.ai.error.clipboardUnavailable"),
|
||||
utteranceID: nil
|
||||
)
|
||||
return
|
||||
}
|
||||
guard let conversationID = state.aiSession.conversationID else { return }
|
||||
let disposition = flow.submitAIQuestion(
|
||||
text: prompt,
|
||||
conversationID: conversationID
|
||||
)
|
||||
if case .rejected(let rejection) = disposition {
|
||||
state.aiSession.fail(message(for: rejection), utteranceID: nil)
|
||||
}
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
guard state.aiSession.isBusy else { return }
|
||||
flow.cancelAIRecording()
|
||||
|
||||
@@ -34,7 +34,7 @@ public struct AppGroupPersistor {
|
||||
// Both engines always polish; ignore legacy off/transcribe modeId.
|
||||
state.mode = .polish
|
||||
state.engineMode = store.engineMode
|
||||
// v0.2.1 follow-up: only the target locale is persisted —
|
||||
// Only the target locale is persisted;
|
||||
// `translationEnabled` is derived from it. Hydrate once at
|
||||
// startup; `refreshRuntimeFlags` keeps the chip in sync while
|
||||
// the keyboard stays open.
|
||||
@@ -47,23 +47,26 @@ public struct AppGroupPersistor {
|
||||
applyAPIKeyAvailability(store: store, into: state)
|
||||
|
||||
#if DEBUG
|
||||
// Print a masked view of the live App Group config so we can see
|
||||
// from the device console exactly what the keyboard extension
|
||||
// actually sees (and whether it agrees with the main App).
|
||||
let key = store.apiKey
|
||||
let masked: String
|
||||
if key.count > 8 {
|
||||
masked = "\(key.prefix(4))…\(key.suffix(4)) (\(key.count) chars)"
|
||||
} else if key.isEmpty {
|
||||
masked = "<empty>"
|
||||
} else {
|
||||
masked = "<\(key.count) chars>"
|
||||
// Log only credential availability and the base URL origin. Never put
|
||||
// credential fragments or URL path/query/userinfo into device logs.
|
||||
let credentialStatus: String
|
||||
switch Keychain.apiKeyOutcome(for: store.providerId, preferICloudSync: true) {
|
||||
case .found(let value):
|
||||
credentialStatus = value.isEmpty ? "empty" : "configured"
|
||||
case .notFound:
|
||||
credentialStatus = store.apiKey.isEmpty ? "empty" : "configured"
|
||||
case .unavailable:
|
||||
credentialStatus = "keychainUnavailable"
|
||||
}
|
||||
let components = URLComponents(string: store.baseURL)
|
||||
let baseURLOrigin = components?.scheme.flatMap { scheme in
|
||||
components?.host.map { host in "\(scheme)://\(host)" }
|
||||
} ?? "<invalid>"
|
||||
print("""
|
||||
🔍 [AppGroupPersistor.load]
|
||||
providerId = \(store.providerId)
|
||||
baseURL = \(store.baseURL)
|
||||
apiKey = \(masked)
|
||||
baseURLOrigin = \(baseURLOrigin)
|
||||
credential = \(credentialStatus)
|
||||
model = \(store.model)
|
||||
modeId = \(store.modeId)
|
||||
localeId = \(store.localeId)
|
||||
@@ -136,11 +139,11 @@ public struct AppGroupPersistor {
|
||||
AppGroupStore().setEngineMode(engineMode)
|
||||
}
|
||||
|
||||
/// v0.2.1: persist translation target locale id (e.g. `"en"`,
|
||||
/// Persist translation target locale id (e.g. `"en"`,
|
||||
/// `"ja"`, or `TranslationLanguageCatalog.offLocaleId`). The
|
||||
/// chip / picker call this through `KeyboardState.setTranslationTargetLocaleId`.
|
||||
///
|
||||
/// v0.2.1 follow-up: removed `persist(translationEnabled:)` — the
|
||||
/// There is no `persist(translationEnabled:)` because the
|
||||
/// enabled state is derived from the locale id, so callers only
|
||||
/// need to write the locale. Keeping the legacy Bool overload
|
||||
/// around would have implied that there's a separate on/off
|
||||
|
||||
@@ -8,20 +8,51 @@ import Foundation
|
||||
import UIKit
|
||||
import OSGKeyboardShared
|
||||
|
||||
@MainActor
|
||||
protocol ClipboardPasteboardProviding: AnyObject {
|
||||
var changeCount: Int { get }
|
||||
var hasStrings: Bool { get }
|
||||
var string: String? { get }
|
||||
}
|
||||
|
||||
@MainActor
|
||||
final class SystemClipboardPasteboard: ClipboardPasteboardProviding {
|
||||
var changeCount: Int { UIPasteboard.general.changeCount }
|
||||
var hasStrings: Bool { UIPasteboard.general.hasStrings }
|
||||
var string: String? { UIPasteboard.general.string }
|
||||
}
|
||||
|
||||
@MainActor
|
||||
final class ClipboardCaptureCoordinator {
|
||||
/// Universal Clipboard may synchronously fetch from another device for
|
||||
/// seconds. System pasteboard reads must never block keyboard presentation.
|
||||
private static let readQueue = DispatchQueue(
|
||||
label: "com.osgkeyboard.clipboard.read",
|
||||
qos: .utility
|
||||
)
|
||||
private static let pollInterval: TimeInterval = 0.8
|
||||
|
||||
private let state: KeyboardState
|
||||
private let history: ClipboardHistoryStore
|
||||
private let pasteboard: ClipboardPasteboardProviding
|
||||
private var pollTimer: Timer?
|
||||
private var isSecureProvider: () -> Bool = { false }
|
||||
private var hasFullAccessProvider: () -> Bool = { false }
|
||||
/// Ephemeral only: leaving a secure field must not resurrect old body text.
|
||||
private var secureFieldSuppressedChangeCount: Int?
|
||||
private var isSecureEntryActive = false
|
||||
private var isSampling = false
|
||||
private var forcesNextSample = true
|
||||
private var isKeyboardVisible = false
|
||||
|
||||
init(
|
||||
state: KeyboardState,
|
||||
history: ClipboardHistoryStore = .shared
|
||||
history: ClipboardHistoryStore = .shared,
|
||||
pasteboard: ClipboardPasteboardProviding = SystemClipboardPasteboard()
|
||||
) {
|
||||
self.state = state
|
||||
self.history = history
|
||||
self.pasteboard = pasteboard
|
||||
}
|
||||
|
||||
func configure(
|
||||
@@ -33,22 +64,54 @@ final class ClipboardCaptureCoordinator {
|
||||
}
|
||||
|
||||
func keyboardDidAppear() {
|
||||
isKeyboardVisible = true
|
||||
history.reload()
|
||||
refreshSuggestionFromStore()
|
||||
captureIfNeeded(force: true)
|
||||
// A suggestion belongs to one keyboard presentation. Clear any
|
||||
// presentation state left behind by a reused extension controller.
|
||||
endCurrentSuggestion()
|
||||
forcesNextSample = true
|
||||
// Delay the system pasteboard read until the first poll tick. A
|
||||
// Universal Clipboard fetch or paste alert during the appear sequence
|
||||
// can otherwise freeze the keyboard before SwiftUI draws.
|
||||
if !(pasteboard is SystemClipboardPasteboard) {
|
||||
captureIfNeeded(forceRead: true)
|
||||
}
|
||||
startPolling()
|
||||
}
|
||||
|
||||
func keyboardWillDisappear() {
|
||||
isKeyboardVisible = false
|
||||
stopPolling()
|
||||
// A1 policy: closing the keyboard ends this generation's suggestion.
|
||||
endCurrentSuggestion()
|
||||
}
|
||||
|
||||
func refreshFlagsFromStore() {
|
||||
// Called from App Group poll — suggestion visibility may change.
|
||||
refreshSuggestionFromStore()
|
||||
// Settings changes may hide the active suggestion, but enabling the
|
||||
// strip must wait for a new pasteboard generation.
|
||||
if !state.clipboardHistoryEnabled || !state.clipboardCandidateBarEnabled {
|
||||
endCurrentSuggestion()
|
||||
}
|
||||
}
|
||||
|
||||
func secureEntryDidChange(isSecure: Bool) {
|
||||
if isSecure {
|
||||
isSecureEntryActive = true
|
||||
secureFieldSuppressedChangeCount = pasteboard.changeCount
|
||||
} else if isSecureEntryActive {
|
||||
// Capture the latest generation once more on exit so a pasteboard
|
||||
// change near the secure-field transition cannot be persisted.
|
||||
secureFieldSuppressedChangeCount = pasteboard.changeCount
|
||||
isSecureEntryActive = false
|
||||
} else {
|
||||
return
|
||||
}
|
||||
endCurrentSuggestion()
|
||||
state.clipboardOverlay = .none
|
||||
}
|
||||
|
||||
func openPanelFromTopButton() {
|
||||
guard state.canShowClipboardEntry else { return }
|
||||
if state.clipboardHistoryEnabled {
|
||||
history.reload()
|
||||
state.clipboardOverlay = .historyPanel
|
||||
@@ -62,15 +125,15 @@ final class ClipboardCaptureCoordinator {
|
||||
}
|
||||
|
||||
func noteUserDidInputText() {
|
||||
clearSuggestion(persistDismiss: false)
|
||||
endCurrentSuggestion()
|
||||
}
|
||||
|
||||
func dismissSuggestion() {
|
||||
history.dismissSuggestion(forChangeCount: state.clipboardSuggestionChangeCount)
|
||||
clearSuggestion(persistDismiss: true)
|
||||
endCurrentSuggestion()
|
||||
}
|
||||
|
||||
func insertText(_ text: String, via insert: (String) -> Void) {
|
||||
guard state.canShowClipboardEntry else { return }
|
||||
insert(text)
|
||||
// Tapping a suggestion (or history row that shares this path) must not
|
||||
// resurface the same clipboard changeCount until the pasteboard changes.
|
||||
@@ -79,24 +142,28 @@ final class ClipboardCaptureCoordinator {
|
||||
}
|
||||
|
||||
func clearHistory() {
|
||||
endCurrentSuggestion()
|
||||
history.clearAll()
|
||||
clearSuggestion(persistDismiss: false)
|
||||
}
|
||||
|
||||
func deleteEntry(id: UUID) {
|
||||
let deletedChangeCount = history.entries.first(where: { $0.id == id })?.changeCount
|
||||
history.remove(id: id)
|
||||
refreshSuggestionFromStore()
|
||||
if deletedChangeCount == state.clipboardSuggestionChangeCount {
|
||||
endCurrentSuggestion()
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Capture
|
||||
|
||||
private func startPolling() {
|
||||
stopPolling()
|
||||
let timer = Timer(timeInterval: 0.8, repeats: true) { [weak self] _ in
|
||||
let timer = Timer(timeInterval: Self.pollInterval, repeats: true) { [weak self] _ in
|
||||
Task { @MainActor in
|
||||
self?.captureIfNeeded(force: false)
|
||||
self?.captureIfNeeded()
|
||||
}
|
||||
}
|
||||
timer.tolerance = Self.pollInterval / 4
|
||||
RunLoop.main.add(timer, forMode: .common)
|
||||
pollTimer = timer
|
||||
}
|
||||
@@ -106,83 +173,170 @@ final class ClipboardCaptureCoordinator {
|
||||
pollTimer = nil
|
||||
}
|
||||
|
||||
private func captureIfNeeded(force: Bool) {
|
||||
func captureIfNeeded(forceRead: Bool = false) {
|
||||
guard state.clipboardHistoryEnabled else {
|
||||
clearSuggestion(persistDismiss: false)
|
||||
endCurrentSuggestion()
|
||||
return
|
||||
}
|
||||
#if DEBUG
|
||||
// What's New demo seeds history itself — never touch the pasteboard
|
||||
// (avoids the simulator “允许粘贴” alert mid-recording).
|
||||
if WhatsNewDemoScenario.peek() != nil || WhatsNewDemoScenario.isPlaying() {
|
||||
return
|
||||
}
|
||||
#endif
|
||||
guard hasFullAccessProvider() else { return }
|
||||
guard !isSecureProvider() else { return }
|
||||
|
||||
let pasteboard = UIPasteboard.general
|
||||
let changeCount = pasteboard.changeCount
|
||||
if !force, changeCount == history.lastObservedChangeCount {
|
||||
refreshSuggestionFromStore()
|
||||
guard !isSecureProvider() else {
|
||||
secureEntryDidChange(isSecure: true)
|
||||
return
|
||||
}
|
||||
|
||||
if pasteboard is SystemClipboardPasteboard {
|
||||
beginSystemSample(forceRead: forceRead || forcesNextSample)
|
||||
forcesNextSample = false
|
||||
return
|
||||
}
|
||||
captureInjectedPasteboard(forceRead: forceRead)
|
||||
}
|
||||
|
||||
/// Synchronous path retained for deterministic tests and injected fakes.
|
||||
/// Production always uses `beginSystemSample` below.
|
||||
private func captureInjectedPasteboard(forceRead: Bool) {
|
||||
let changeCount = pasteboard.changeCount
|
||||
if ClipboardHistoryPolicy.shouldSuppressCapture(
|
||||
changeCount: changeCount,
|
||||
secureFieldSuppressedChangeCount: secureFieldSuppressedChangeCount
|
||||
) {
|
||||
history.lastObservedChangeCount = changeCount
|
||||
clearSuggestion()
|
||||
return
|
||||
}
|
||||
secureFieldSuppressedChangeCount = nil
|
||||
let isCurrentGeneration = changeCount == history.lastObservedChangeCount
|
||||
if isCurrentGeneration && !forceRead {
|
||||
return
|
||||
}
|
||||
|
||||
// A new generation replaces any previous transient suggestion,
|
||||
// including generations that contain no acceptable text.
|
||||
clearSuggestion()
|
||||
|
||||
// Prefer hasStrings peek before reading body (reduces empty reads).
|
||||
guard pasteboard.hasStrings else {
|
||||
history.lastObservedChangeCount = changeCount
|
||||
refreshSuggestionFromStore()
|
||||
return
|
||||
}
|
||||
|
||||
let raw = pasteboard.string
|
||||
// The forced appearance read exists only to establish/refresh iOS
|
||||
// paste permission. It must not reinsert or republish old content.
|
||||
if isCurrentGeneration {
|
||||
return
|
||||
}
|
||||
if let entry = history.ingest(rawText: raw, changeCount: changeCount) {
|
||||
updateSuggestion(with: entry, changeCount: changeCount)
|
||||
} else {
|
||||
history.lastObservedChangeCount = changeCount
|
||||
refreshSuggestionFromStore()
|
||||
}
|
||||
}
|
||||
|
||||
private func refreshSuggestionFromStore() {
|
||||
guard state.clipboardHistoryEnabled,
|
||||
state.clipboardCandidateBarEnabled,
|
||||
!isSecureProvider(),
|
||||
let newest = history.newestEntry
|
||||
else {
|
||||
clearSuggestion(persistDismiss: false)
|
||||
private struct Sample: Sendable {
|
||||
let changeCount: Int
|
||||
let hasStrings: Bool
|
||||
let text: String?
|
||||
}
|
||||
|
||||
private func beginSystemSample(forceRead: Bool) {
|
||||
guard !isSampling else { return }
|
||||
isSampling = true
|
||||
let lastObserved = history.lastObservedChangeCount
|
||||
|
||||
Self.readQueue.async {
|
||||
let pasteboard = UIPasteboard.general
|
||||
let changeCount = pasteboard.changeCount
|
||||
guard forceRead || changeCount != lastObserved else {
|
||||
Task { @MainActor [weak self] in
|
||||
self?.isSampling = false
|
||||
}
|
||||
return
|
||||
}
|
||||
let hasStrings = pasteboard.hasStrings
|
||||
let sample = Sample(
|
||||
changeCount: changeCount,
|
||||
hasStrings: hasStrings,
|
||||
text: hasStrings ? pasteboard.string : nil
|
||||
)
|
||||
Task { @MainActor [weak self] in
|
||||
self?.finishSystemSample(sample)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func finishSystemSample(_ sample: Sample) {
|
||||
isSampling = false
|
||||
guard isKeyboardVisible,
|
||||
state.clipboardHistoryEnabled,
|
||||
hasFullAccessProvider(),
|
||||
!isSecureProvider()
|
||||
else { return }
|
||||
|
||||
let changeCount = sample.changeCount
|
||||
if ClipboardHistoryPolicy.shouldSuppressCapture(
|
||||
changeCount: changeCount,
|
||||
secureFieldSuppressedChangeCount: secureFieldSuppressedChangeCount
|
||||
) {
|
||||
history.lastObservedChangeCount = changeCount
|
||||
clearSuggestion()
|
||||
return
|
||||
}
|
||||
let changeCount = newest.changeCount ?? history.lastObservedChangeCount
|
||||
guard history.shouldShowSuggestion(
|
||||
forChangeCount: changeCount,
|
||||
candidateBarEnabled: state.clipboardCandidateBarEnabled,
|
||||
historyEnabled: state.clipboardHistoryEnabled
|
||||
) else {
|
||||
clearSuggestion(persistDismiss: false)
|
||||
secureFieldSuppressedChangeCount = nil
|
||||
let isCurrentGeneration = changeCount == history.lastObservedChangeCount
|
||||
|
||||
// A new generation replaces any previous transient suggestion,
|
||||
// including generations that contain no acceptable text.
|
||||
clearSuggestion()
|
||||
guard sample.hasStrings else {
|
||||
history.lastObservedChangeCount = changeCount
|
||||
return
|
||||
}
|
||||
// Don't resurrect a strip the user already dismissed this session
|
||||
// unless changeCount advanced (handled in ingest).
|
||||
if state.clipboardSuggestionText == nil,
|
||||
let dismissed = history.suggestionDismissedChangeCount,
|
||||
dismissed == changeCount {
|
||||
return
|
||||
// Forced appearance reads establish iOS paste permission only. Never
|
||||
// republish content from an already observed generation.
|
||||
guard !isCurrentGeneration else { return }
|
||||
|
||||
if let entry = history.ingest(rawText: sample.text, changeCount: changeCount) {
|
||||
updateSuggestion(with: entry, changeCount: changeCount)
|
||||
} else {
|
||||
history.lastObservedChangeCount = changeCount
|
||||
}
|
||||
state.clipboardSuggestionText = newest.text
|
||||
state.clipboardSuggestionChangeCount = changeCount
|
||||
}
|
||||
|
||||
private func updateSuggestion(with entry: ClipboardHistoryEntry, changeCount: Int) {
|
||||
guard state.clipboardCandidateBarEnabled else {
|
||||
clearSuggestion(persistDismiss: false)
|
||||
guard state.canShowClipboardEntry,
|
||||
state.clipboardCandidateBarEnabled,
|
||||
changeCount != secureFieldSuppressedChangeCount
|
||||
else {
|
||||
clearSuggestion()
|
||||
return
|
||||
}
|
||||
// Already used/dismissed this pasteboard generation — keep it hidden.
|
||||
if history.suggestionDismissedChangeCount == changeCount {
|
||||
clearSuggestion(persistDismiss: false)
|
||||
clearSuggestion()
|
||||
return
|
||||
}
|
||||
state.clipboardSuggestionText = entry.text
|
||||
state.clipboardSuggestionChangeCount = changeCount
|
||||
}
|
||||
|
||||
private func clearSuggestion(persistDismiss: Bool) {
|
||||
if persistDismiss {
|
||||
// already written in dismissSuggestion
|
||||
private func endCurrentSuggestion() {
|
||||
history.dismissSuggestion(forChangeCount: state.clipboardSuggestionChangeCount)
|
||||
clearSuggestion()
|
||||
}
|
||||
|
||||
private func clearSuggestion() {
|
||||
guard state.clipboardSuggestionText != nil
|
||||
|| state.clipboardSuggestionChangeCount != nil
|
||||
else {
|
||||
return
|
||||
}
|
||||
state.clipboardSuggestionText = nil
|
||||
state.clipboardSuggestionChangeCount = nil
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
// EditHintScheduler.swift
|
||||
// OSGKeyboard · Keyboard Extension
|
||||
//
|
||||
// Owns the edit-hint lifetime so every producer shares one expiration order.
|
||||
|
||||
import Foundation
|
||||
import OSGKeyboardShared
|
||||
|
||||
@MainActor
|
||||
final class EditHintScheduler {
|
||||
typealias Sleeper = @MainActor (Duration) async -> Void
|
||||
|
||||
private let state: KeyboardState
|
||||
private let sleeper: Sleeper
|
||||
private var task: Task<Void, Never>?
|
||||
private var generation: UInt64 = 0
|
||||
|
||||
init(
|
||||
state: KeyboardState,
|
||||
sleeper: @escaping Sleeper = { duration in
|
||||
try? await Task.sleep(for: duration)
|
||||
}
|
||||
) {
|
||||
self.state = state
|
||||
self.sleeper = sleeper
|
||||
}
|
||||
|
||||
func show(message: String, isPositive: Bool, duration: Duration) {
|
||||
let scheduledGeneration = advanceGeneration()
|
||||
task?.cancel()
|
||||
|
||||
state.editHint = message
|
||||
state.editHintIsPositive = isPositive
|
||||
|
||||
let sleeper = sleeper
|
||||
task = Task { @MainActor [weak self, sleeper] in
|
||||
await sleeper(duration)
|
||||
guard let self, self.generation == scheduledGeneration else {
|
||||
return
|
||||
}
|
||||
self.task = nil
|
||||
self.clearHint()
|
||||
}
|
||||
}
|
||||
|
||||
func clearPositive() {
|
||||
guard state.editHintIsPositive else { return }
|
||||
advanceGeneration()
|
||||
task?.cancel()
|
||||
task = nil
|
||||
clearHint()
|
||||
}
|
||||
|
||||
func invalidate() {
|
||||
advanceGeneration()
|
||||
task?.cancel()
|
||||
task = nil
|
||||
clearHint()
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
private func advanceGeneration() -> UInt64 {
|
||||
generation &+= 1
|
||||
return generation
|
||||
}
|
||||
|
||||
private func clearHint() {
|
||||
state.editHint = nil
|
||||
state.editHintIsPositive = false
|
||||
}
|
||||
}
|
||||
@@ -316,10 +316,6 @@ final class KeyboardFlowCoordinator {
|
||||
snapshotReason: readySnapshot?.reason
|
||||
)
|
||||
state.flowSessionActive = sessionActive
|
||||
state.debugPendingFlowStart = isPendingFlowStart
|
||||
state.debugFlowRecording = isFlowRecording
|
||||
state.debugAwaitingFlowResult = isAwaitingFlowResult
|
||||
state.debugHasFullAccess = hasFullAccess()
|
||||
state.micVoiceAvailability = MicVoiceAvailabilityResolver.resolve(
|
||||
phase: state.phase,
|
||||
micDisabled: state.micDisabled,
|
||||
@@ -641,6 +637,17 @@ final class KeyboardFlowCoordinator {
|
||||
startUtterance(.aiQuestion(conversationID: conversationID))
|
||||
}
|
||||
|
||||
func submitAIQuestion(
|
||||
text: String,
|
||||
conversationID: UUID
|
||||
) -> FlowUtteranceStartDisposition {
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return .rejected(.pipelineBusy) }
|
||||
return startUtterance(
|
||||
.aiQuestion(conversationID: conversationID, prefilledQuestion: trimmed)
|
||||
)
|
||||
}
|
||||
|
||||
func stopAIRecording() {
|
||||
guard currentUtteranceRequest?.isAIQuestion == true else { return }
|
||||
pressEnded()
|
||||
@@ -1223,7 +1230,7 @@ final class KeyboardFlowCoordinator {
|
||||
"status=\(result.status.rawValue) "
|
||||
+ "kind=\(result.errorKind?.rawValue ?? "none") "
|
||||
+ "utterance=\(result.utteranceId.uuidString.prefix(8)) "
|
||||
+ "message=\(result.text ?? "nil")"
|
||||
+ "messageLen=\(result.text?.count ?? 0)"
|
||||
)
|
||||
isAwaitingFlowResult = false
|
||||
stopFlowWatchdog()
|
||||
@@ -1738,6 +1745,26 @@ final class KeyboardFlowCoordinator {
|
||||
}
|
||||
FlowSessionBridge.setPendingKeyboardUtteranceId(currentUtteranceId)
|
||||
lastStoppedUtteranceId = nil
|
||||
|
||||
// Prefilled AI hint: skip mic / ASR and ask the host to answer text.
|
||||
if currentUtteranceRequest?.isAIQuestion == true,
|
||||
let question = currentUtteranceRequest?.aiQuestionText?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!question.isEmpty {
|
||||
writeSubmitAIQuestion(question)
|
||||
isFlowRecording = false
|
||||
isAwaitingFlowResult = true
|
||||
state.lastTranscript = question
|
||||
state.phase = .processing
|
||||
if let currentUtteranceId {
|
||||
onAIRecognitionStarted(currentUtteranceId)
|
||||
}
|
||||
startFlowResultWatchdog()
|
||||
recomputeMicVoiceAvailability()
|
||||
traceState("startFlowRecording.submitAIQuestion")
|
||||
return
|
||||
}
|
||||
|
||||
writeCommand(.startRecording)
|
||||
isFlowRecording = true
|
||||
state.lastTranscript = ""
|
||||
@@ -1759,6 +1786,37 @@ final class KeyboardFlowCoordinator {
|
||||
traceState("startFlowRecording.started")
|
||||
}
|
||||
|
||||
private func writeSubmitAIQuestion(_ text: String) {
|
||||
guard let activeSessionId, let currentUtteranceId else { return }
|
||||
let command = FlowCommand(
|
||||
sessionId: activeSessionId,
|
||||
utteranceId: currentUtteranceId,
|
||||
commandSeq: nextCommandSeq(),
|
||||
action: .submitAIQuestion,
|
||||
localeId: state.localeId,
|
||||
utteranceMode: .aiQuestion,
|
||||
aiConversationID: currentUtteranceRequest?.aiConversationID,
|
||||
aiQuestionText: text,
|
||||
startDeadlineAt: currentStartDeadlineAt
|
||||
)
|
||||
FlowSessionBridge.writeCommand(command)
|
||||
if let currentStartDeadlineAt {
|
||||
FlowSessionBridge.writeStartTransaction(
|
||||
FlowStartTransaction(
|
||||
sessionID: activeSessionId,
|
||||
utteranceID: currentUtteranceId,
|
||||
deadlineAt: currentStartDeadlineAt,
|
||||
phase: .issued
|
||||
)
|
||||
)
|
||||
}
|
||||
FlowTrace.keyboard(
|
||||
"command.submitAIQuestion",
|
||||
"seq=\(command.commandSeq) utterance=\(currentUtteranceId.uuidString.prefix(8)) "
|
||||
+ "chars=\(text.count)"
|
||||
)
|
||||
}
|
||||
|
||||
private func startUtteranceCountdown() {
|
||||
utteranceStartedAt = Date().timeIntervalSince1970
|
||||
state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration)
|
||||
@@ -2012,7 +2070,7 @@ final class KeyboardFlowCoordinator {
|
||||
"via=resultWatchdog status=\(result.status.rawValue) "
|
||||
+ "kind=\(error.kind.rawValue) "
|
||||
+ "utterance=\(result.utteranceId.uuidString.prefix(8)) "
|
||||
+ "message=\(error.message)"
|
||||
+ "messageLen=\(error.message.count)"
|
||||
)
|
||||
self.state.phase = .error(
|
||||
.fromFlowTranscription(error),
|
||||
|
||||
@@ -21,6 +21,7 @@ final class KeyboardTextInserter {
|
||||
private let fieldContextProvider: () -> FlowFieldContext?
|
||||
private let selectedText: () -> String?
|
||||
private let scheduleAutoClearError: () -> Void
|
||||
private unowned let editHintScheduler: EditHintScheduler
|
||||
|
||||
/// Exact string last inserted through this inserter — dictation, AI answer,
|
||||
/// edit result or clipboard paste (including any word-boundary separator).
|
||||
@@ -33,7 +34,6 @@ final class KeyboardTextInserter {
|
||||
private var redoContextBefore: String?
|
||||
private let extensionInstanceID = UUID()
|
||||
private var lastEditUndo: PendingTextEditTransaction?
|
||||
private var editHintTask: Task<Void, Never>?
|
||||
/// Suppresses availability refresh while we walk `deleteBackward`
|
||||
/// for undo, so intermediate contexts don't flicker the button.
|
||||
private var isUndoing = false
|
||||
@@ -45,7 +45,8 @@ final class KeyboardTextInserter {
|
||||
contextBeforeInput: @escaping () -> String?,
|
||||
fieldContextProvider: @escaping () -> FlowFieldContext?,
|
||||
selectedText: @escaping () -> String?,
|
||||
scheduleAutoClearError: @escaping () -> Void
|
||||
scheduleAutoClearError: @escaping () -> Void,
|
||||
editHintScheduler: EditHintScheduler
|
||||
) {
|
||||
self.state = state
|
||||
self.insertText = insertText
|
||||
@@ -54,6 +55,7 @@ final class KeyboardTextInserter {
|
||||
self.fieldContextProvider = fieldContextProvider
|
||||
self.selectedText = selectedText
|
||||
self.scheduleAutoClearError = scheduleAutoClearError
|
||||
self.editHintScheduler = editHintScheduler
|
||||
}
|
||||
|
||||
func handleFlowTranscript(
|
||||
@@ -510,28 +512,16 @@ final class KeyboardTextInserter {
|
||||
extensionInstanceID: extensionInstanceID
|
||||
)
|
||||
)
|
||||
editHintTask?.cancel()
|
||||
let hint = ExtL10n.string("keyboard.edit.hint.available")
|
||||
state.editHint = hint
|
||||
state.editHintIsPositive = true
|
||||
editHintTask = Task { @MainActor [weak self] in
|
||||
try? await Task.sleep(nanoseconds: 10_000_000_000)
|
||||
guard !Task.isCancelled,
|
||||
self?.state.editHint == hint,
|
||||
self?.state.editHintIsPositive == true else {
|
||||
return
|
||||
}
|
||||
self?.state.editHint = nil
|
||||
self?.state.editHintIsPositive = false
|
||||
}
|
||||
editHintScheduler.show(
|
||||
message: hint,
|
||||
isPositive: true,
|
||||
duration: .seconds(10)
|
||||
)
|
||||
}
|
||||
|
||||
private func clearEditHintIfPositive() {
|
||||
guard state.editHintIsPositive else { return }
|
||||
editHintTask?.cancel()
|
||||
editHintTask = nil
|
||||
state.editHint = nil
|
||||
state.editHintIsPositive = false
|
||||
editHintScheduler.clearPositive()
|
||||
}
|
||||
|
||||
private func clearLastInsertion() {
|
||||
|
||||
@@ -15,7 +15,7 @@ final class LastInputEditCoordinator {
|
||||
private let stopFlow: () -> Void
|
||||
private let abortFlow: () -> Void
|
||||
private let acknowledge: (FlowAck.DeliveryOutcome) -> Void
|
||||
private var hintTask: Task<Void, Never>?
|
||||
private unowned let editHintScheduler: EditHintScheduler
|
||||
private var activeUtteranceID: UUID?
|
||||
private var reviewedUtteranceID: UUID?
|
||||
private var reviewedRevision: Int64?
|
||||
@@ -26,7 +26,8 @@ final class LastInputEditCoordinator {
|
||||
beginFlow: @escaping (EditableInputReference) -> FlowUtteranceStartDisposition,
|
||||
stopFlow: @escaping () -> Void,
|
||||
abortFlow: @escaping () -> Void,
|
||||
acknowledge: @escaping (FlowAck.DeliveryOutcome) -> Void
|
||||
acknowledge: @escaping (FlowAck.DeliveryOutcome) -> Void,
|
||||
editHintScheduler: EditHintScheduler
|
||||
) {
|
||||
self.state = state
|
||||
self.textInserter = textInserter
|
||||
@@ -34,6 +35,7 @@ final class LastInputEditCoordinator {
|
||||
self.stopFlow = stopFlow
|
||||
self.abortFlow = abortFlow
|
||||
self.acknowledge = acknowledge
|
||||
self.editHintScheduler = editHintScheduler
|
||||
}
|
||||
|
||||
func begin() {
|
||||
@@ -222,35 +224,14 @@ final class LastInputEditCoordinator {
|
||||
reviewedRevision = nil
|
||||
}
|
||||
|
||||
func showAvailabilityHintAfterDictation() {
|
||||
guard textInserter.editableReference() != nil else { return }
|
||||
showHint(
|
||||
ExtL10n.string("keyboard.edit.hint.available"),
|
||||
isPositive: true,
|
||||
durationNanoseconds: 10_000_000_000
|
||||
private func showHint(_ message: String) {
|
||||
editHintScheduler.show(
|
||||
message: message,
|
||||
isPositive: false,
|
||||
duration: .milliseconds(2_500)
|
||||
)
|
||||
}
|
||||
|
||||
private func showHint(
|
||||
_ message: String,
|
||||
isPositive: Bool = false,
|
||||
durationNanoseconds: UInt64 = 2_500_000_000
|
||||
) {
|
||||
hintTask?.cancel()
|
||||
state.editHint = message
|
||||
state.editHintIsPositive = isPositive
|
||||
hintTask = Task { @MainActor [weak self] in
|
||||
try? await Task.sleep(nanoseconds: durationNanoseconds)
|
||||
guard !Task.isCancelled,
|
||||
self?.state.editHint == message,
|
||||
self?.state.editHintIsPositive == isPositive else {
|
||||
return
|
||||
}
|
||||
self?.state.editHint = nil
|
||||
self?.state.editHintIsPositive = false
|
||||
}
|
||||
}
|
||||
|
||||
private func startFailureMessage(
|
||||
for disposition: FlowUtteranceStartDisposition
|
||||
) -> String {
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
// WhatsNewDemoDriver.swift
|
||||
// OSGKeyboard · Keyboard Extension (DEBUG-only)
|
||||
//
|
||||
// Plays a scripted What's New timeline on the **real** keyboard surface while
|
||||
// a Notes-like host sits underneath. Mutates the host document via the
|
||||
// textDocumentProxy so the clip shows a closed loop (insert / replace).
|
||||
|
||||
#if DEBUG
|
||||
import Foundation
|
||||
import OSGKeyboardShared
|
||||
|
||||
@MainActor
|
||||
enum WhatsNewDemoDriver {
|
||||
private static var running = false
|
||||
private static var activeTask: Task<Void, Never>?
|
||||
|
||||
/// Document mutations + Return for AI “发送”.
|
||||
struct HostHooks {
|
||||
var insertText: (String) -> Void
|
||||
var deleteBackward: () -> Void
|
||||
var contextBeforeInput: () -> String?
|
||||
var performReturn: () -> Void
|
||||
}
|
||||
|
||||
/// Keyboard extensions outlive host relaunches — always allow a fresh arm.
|
||||
static func resetForNewPresentation() {
|
||||
activeTask?.cancel()
|
||||
activeTask = nil
|
||||
running = false
|
||||
WhatsNewDemoScenario.finishPlaying()
|
||||
}
|
||||
|
||||
static func startIfNeeded(state: KeyboardState, host: HostHooks) {
|
||||
// Peek first so a brief appear/disappear does not burn the arm.
|
||||
guard WhatsNewDemoScenario.peek() != nil else { return }
|
||||
// A still-running timeline from a previous host launch must not block
|
||||
// the next What's New recording.
|
||||
if running {
|
||||
resetForNewPresentation()
|
||||
}
|
||||
running = true
|
||||
activeTask = Task { @MainActor in
|
||||
// Wait for first layout / surface mount over the host.
|
||||
try? await Task.sleep(nanoseconds: 900_000_000)
|
||||
guard !Task.isCancelled else {
|
||||
running = false
|
||||
return
|
||||
}
|
||||
guard let armed = WhatsNewDemoScenario.consume() else {
|
||||
running = false
|
||||
return
|
||||
}
|
||||
OSGDiag.log(
|
||||
"WhatsNewDemo start scenario=\(armed.scenario.rawValue) lang=\(armed.language.rawValue)",
|
||||
category: "boot"
|
||||
)
|
||||
polishDemoChrome(state)
|
||||
switch armed.scenario {
|
||||
case .edit:
|
||||
await runEdit(
|
||||
state: state,
|
||||
host: host,
|
||||
original: armed.seedText,
|
||||
language: armed.language
|
||||
)
|
||||
case .ai:
|
||||
await runAI(state: state, host: host, language: armed.language)
|
||||
case .clipboard:
|
||||
await runClipboard(state: state, host: host, language: armed.language)
|
||||
}
|
||||
if !Task.isCancelled {
|
||||
WhatsNewDemoScenario.finishPlaying()
|
||||
}
|
||||
running = false
|
||||
activeTask = nil
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Edit last input
|
||||
|
||||
private static func runEdit(
|
||||
state: KeyboardState,
|
||||
host: HostHooks,
|
||||
original: String,
|
||||
language: WhatsNewDemoScenario.Language
|
||||
) async {
|
||||
let edited = language == .en
|
||||
? "Hi everyone — we'll hold a planning discussion in Conference Room A at 3pm tomorrow. Please be on time."
|
||||
: "各位好,明天下午三点在 A 会议室召开方案讨论会,请准时参加。"
|
||||
let reference = EditableInputReference(
|
||||
displayText: original,
|
||||
insertedText: original,
|
||||
postInsertionFingerprint: nil,
|
||||
extensionInstanceID: UUID()
|
||||
)
|
||||
let source = EditSessionSource(reference: reference)
|
||||
let review = EditReview(
|
||||
source: source,
|
||||
resultText: edited,
|
||||
utteranceID: UUID()
|
||||
)
|
||||
|
||||
state.surface = .voice
|
||||
state.editCanReplaceOriginal = true
|
||||
state.phase = .idle
|
||||
ClipboardHistoryStore.shared.clearAll()
|
||||
clearClipboardChrome(state)
|
||||
polishDemoChrome(state)
|
||||
|
||||
// Brief idle so the host note + voice mic are visible together.
|
||||
try? await sleep(1.0)
|
||||
|
||||
state.editSession = .listening(source)
|
||||
state.lastTranscript = ExtL10n.string("keyboard.edit.status.listening")
|
||||
state.level = 0.45
|
||||
for _ in 0..<4 {
|
||||
try? await sleep(0.28)
|
||||
state.level = Double.random(in: 0.25...0.85)
|
||||
polishDemoChrome(state)
|
||||
}
|
||||
|
||||
state.editSession = .processing(source)
|
||||
state.lastTranscript = ExtL10n.string("keyboard.edit.status.processing")
|
||||
try? await sleep(1.0)
|
||||
|
||||
state.editSession = .review(review)
|
||||
state.lastTranscript = ExtL10n.string("keyboard.edit.status.review")
|
||||
try? await sleep(2.4)
|
||||
|
||||
state.editSession = .applying(review)
|
||||
state.lastTranscript = ExtL10n.string("keyboard.edit.status.applying")
|
||||
// Replace host document so the clip closes the loop.
|
||||
replaceHostText(from: original, to: edited, host: host)
|
||||
try? await sleep(1.2)
|
||||
|
||||
state.editSession = .inactive
|
||||
state.editCanReplaceOriginal = false
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
clearClipboardChrome(state)
|
||||
polishDemoChrome(state)
|
||||
try? await sleep(1.2)
|
||||
}
|
||||
|
||||
// MARK: - AI keyboard
|
||||
|
||||
private static func runAI(
|
||||
state: KeyboardState,
|
||||
host: HostHooks,
|
||||
language: WhatsNewDemoScenario.Language
|
||||
) async {
|
||||
let question = language == .en
|
||||
? "Where should I go this weekend?"
|
||||
: "周末去哪儿玩比较合适?"
|
||||
let answer = language == .en
|
||||
? "Try a nearby town day trip: morning walk in a park or old street, afternoon café, then a local dinner."
|
||||
: "可以去近郊走走:上午逛古镇或公园,下午找一家口碑好的咖啡馆休息,傍晚再吃顿当地特色菜。"
|
||||
|
||||
state.surface = .ai
|
||||
state.aiServiceAvailable = true
|
||||
ClipboardHistoryStore.shared.clearAll()
|
||||
clearClipboardChrome(state)
|
||||
polishDemoChrome(state)
|
||||
// Chat host uses returnKeyType=.send; keep role locked for the clip.
|
||||
state.returnKeyRole = .send
|
||||
state.aiSession.enter()
|
||||
try? await sleep(0.9)
|
||||
|
||||
let utteranceID = UUID()
|
||||
state.aiSession.beginPreparing(utteranceID: utteranceID)
|
||||
try? await sleep(0.25)
|
||||
state.aiSession.beginListening(utteranceID: utteranceID)
|
||||
state.level = 0.4
|
||||
for _ in 0..<6 {
|
||||
try? await sleep(0.18)
|
||||
state.level = Double.random(in: 0.25...0.9)
|
||||
state.aiSession.updateTranscript(question, utteranceID: utteranceID)
|
||||
polishDemoChrome(state)
|
||||
}
|
||||
|
||||
state.aiSession.beginRecognizing(utteranceID: utteranceID)
|
||||
try? await sleep(0.4)
|
||||
state.aiSession.beginGenerating(question: question, utteranceID: utteranceID)
|
||||
try? await sleep(0.45)
|
||||
|
||||
let chars = Array(answer)
|
||||
var index = 0
|
||||
while index < chars.count {
|
||||
index = min(chars.count, index + 5)
|
||||
let draft = String(chars.prefix(index))
|
||||
state.aiSession.receivePartialAnswer(draft, utteranceID: utteranceID)
|
||||
try? await sleep(0.1)
|
||||
}
|
||||
state.aiSession.receiveAnswer(answer, utteranceID: utteranceID)
|
||||
polishDemoChrome(state)
|
||||
try? await sleep(1.3)
|
||||
|
||||
// Insert on a new line so seed + answer stay readable in the composer.
|
||||
host.insertText("\n" + answer)
|
||||
state.aiSession.markAnswerInserted(offersSend: true)
|
||||
polishDemoChrome(state)
|
||||
try? await sleep(1.1)
|
||||
|
||||
state.aiSession.markAnswerSent()
|
||||
host.performReturn()
|
||||
// Stay on AI surface so the “已发送” beat is not buried by voice chrome.
|
||||
state.surface = .ai
|
||||
clearClipboardChrome(state)
|
||||
polishDemoChrome(state)
|
||||
try? await sleep(1.4)
|
||||
}
|
||||
|
||||
// MARK: - Clipboard history
|
||||
|
||||
private static func runClipboard(
|
||||
state: KeyboardState,
|
||||
host: HostHooks,
|
||||
language: WhatsNewDemoScenario.Language
|
||||
) async {
|
||||
let samples = language == .en
|
||||
? [
|
||||
"Meeting at 3pm tomorrow",
|
||||
"Room moved to Building A, 3F",
|
||||
"Bring the clicker and the deck"
|
||||
]
|
||||
: [
|
||||
"明天下午三点开会",
|
||||
"会议室改到 A 栋 3 楼",
|
||||
"请带上投影笔和方案文档"
|
||||
]
|
||||
let store = ClipboardHistoryStore.shared
|
||||
store.clearAll()
|
||||
for text in samples.reversed() {
|
||||
_ = store.ingest(rawText: text, changeCount: Int.random(in: 1...9_999))
|
||||
}
|
||||
store.reload()
|
||||
|
||||
// Persist flags so AppGroupPersistor cannot flip history off mid-demo.
|
||||
if let defaults = AppGroup.defaultsIfAvailable {
|
||||
defaults.set(true, forKey: AppGroupConfiguration.Keys.clipboardHistoryEnabled)
|
||||
defaults.set(true, forKey: AppGroupConfiguration.Keys.clipboardCandidateBarEnabled)
|
||||
defaults.synchronize()
|
||||
}
|
||||
|
||||
state.surface = .voice
|
||||
state.clipboardHistoryEnabled = true
|
||||
state.clipboardCandidateBarEnabled = true
|
||||
state.phase = .idle
|
||||
state.clipboardOverlay = .none
|
||||
polishDemoChrome(state)
|
||||
// Suggestion strip first (matches the docs copy).
|
||||
state.clipboardSuggestionText = samples[0]
|
||||
state.clipboardSuggestionChangeCount = 1
|
||||
|
||||
try? await sleep(1.2)
|
||||
polishDemoChrome(state)
|
||||
|
||||
state.clipboardOverlay = .historyPanel
|
||||
try? await sleep(2.6)
|
||||
polishDemoChrome(state)
|
||||
|
||||
// Tap-to-insert: close panel + write into the host “待办:” field.
|
||||
state.clipboardOverlay = .none
|
||||
host.insertText(samples[0])
|
||||
polishDemoChrome(state)
|
||||
try? await sleep(0.9)
|
||||
|
||||
// Leave a fresh suggestion strip visible for the next copy cue.
|
||||
state.clipboardSuggestionText = samples[1]
|
||||
state.clipboardSuggestionChangeCount = 2
|
||||
// Hold suggestion with chrome locked clean (no Flow warning flash).
|
||||
for _ in 0..<8 {
|
||||
polishDemoChrome(state)
|
||||
try? await sleep(0.2)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Host helpers
|
||||
|
||||
private static func polishDemoChrome(_ state: KeyboardState) {
|
||||
state.micDisabledHint = ""
|
||||
state.micVoiceAvailability = .ready
|
||||
}
|
||||
|
||||
private static func clearClipboardChrome(_ state: KeyboardState) {
|
||||
state.clipboardOverlay = .none
|
||||
state.clipboardSuggestionText = nil
|
||||
state.clipboardSuggestionChangeCount = nil
|
||||
}
|
||||
|
||||
private static func replaceHostText(
|
||||
from original: String,
|
||||
to edited: String,
|
||||
host: HostHooks
|
||||
) {
|
||||
// Prefer deleting only the seed suffix so we don't wipe unrelated text.
|
||||
let before = host.contextBeforeInput() ?? ""
|
||||
let deleteCount: Int
|
||||
if before.hasSuffix(original) {
|
||||
deleteCount = original.count
|
||||
} else if !before.isEmpty {
|
||||
deleteCount = before.count
|
||||
} else {
|
||||
deleteCount = original.count
|
||||
}
|
||||
for _ in 0..<deleteCount {
|
||||
host.deleteBackward()
|
||||
}
|
||||
host.insertText(edited)
|
||||
}
|
||||
|
||||
/// ~1.6× slow-mo for screen recording readability (30 fps source).
|
||||
private static func sleep(_ seconds: Double) async throws {
|
||||
try await Task.sleep(nanoseconds: UInt64(seconds * 1.6 * 1_000_000_000))
|
||||
}
|
||||
}
|
||||
#endif
|
||||
Reference in New Issue
Block a user