feat(app): add crash visibility, memory relief, and clipboard reply work
Squashes the working tree that had accumulated on cloud-agent. Two strands of work, kept in one commit at the maintainer's request. Crash and stability (this session): - Subscribe to MetricKit so crash, hang, CPU, and disk-write diagnostics are recorded on device. Keyboard-extension memory kills are the most likely way a user loses the keyboard mid-sentence and iOS does not classify them as crashes, so they never reach App Store Connect. - Upload those reports to POST /v1/analytics/diagnostics from internal builds only (local Debug and TestFlight), reusing the analytics installation identifier and skipping entirely when analytics are off. App Store builds keep everything on device. - Shed the Chinese engine, English lexicon, and in-flight pipeline work at the 40 MiB and 48 MiB budget thresholds instead of waiting for a system memory warning that jetsam often preempts. - Install audio taps with the input node's live format through one shared adaptive resampler, so an audio-route change mid-recording no longer raises an uncatchable format-mismatch exception. Covers the dictation preview and the Mac recorder, which still used the pattern the Flow capture path had already moved away from. - Harden the librime bridge: never pass a nil string across a nonnull boundary when a user dictionary holds malformed UTF-8, null-check the engine API vector, and distrust a candidate count without its array. - Guard the expanded candidate grid against a list that shrinks while the collection view is still laying out against the previous count. - Resolve localized text without the App Group so the screen shown when that container is missing renders its message instead of crashing on launch, which is the one situation it exists to handle. - Gate CI on the App Group and Keychain wiring that the keyboard extension hard-depends on, with a self-test proving the check fails. Clipboard, reply, and semantics (pre-existing working tree): - Unify the reply center, refresh keyboard AI features, and add native event and reminder exporters. - Expand clipboard semantic routing with authored corpora and a product-blind holdout, plus the scripts that build and validate them. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -4,6 +4,7 @@
|
||||
// Samples the general pasteboard on keyboard appear and while visible
|
||||
// (changeCount-driven). Writes accepted text into ClipboardHistoryStore.
|
||||
|
||||
import Combine
|
||||
import Foundation
|
||||
import OSGKeyboardShared
|
||||
import UIKit
|
||||
@@ -45,6 +46,14 @@ final class ClipboardCaptureCoordinator {
|
||||
private var isSampling = false
|
||||
private var forcesNextSample = true
|
||||
private var isKeyboardVisible = false
|
||||
/// OOBE reply / translate practice auto-fires once per feature so the user
|
||||
/// sees the keyboard draft the result without tapping the chip. Tracked per
|
||||
/// feature (the host keeps one session id across every practice step).
|
||||
private var oobeAutoFiredSessionID: UUID?
|
||||
private var oobeAutoFiredFeatures: Set<ManagedGatewayOOBEFeature> = []
|
||||
/// Fires the auto-reply evaluation whenever a fresh semantic analysis lands,
|
||||
/// independent of which keyboard surface (voice / Chinese / English) is shown.
|
||||
private var snapshotObserver: AnyCancellable?
|
||||
|
||||
init(
|
||||
state: KeyboardState,
|
||||
@@ -56,6 +65,17 @@ final class ClipboardCaptureCoordinator {
|
||||
self.history = history
|
||||
self.semanticRanking = semanticRanking
|
||||
self.pasteboard = pasteboard
|
||||
snapshotObserver = semanticRanking.$snapshot
|
||||
.sink { [weak self] snapshot in
|
||||
// `@Published` fires in willSet, so the store's property is not
|
||||
// updated yet; hop to the main actor so the trigger reads the
|
||||
// published snapshot (and satisfies actor isolation).
|
||||
guard snapshot != nil else { return }
|
||||
Task { @MainActor in
|
||||
self?.autoTriggerActionIfNeeded()
|
||||
self?.presentAutoReplyGuideIfNeeded()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func configure(
|
||||
@@ -86,6 +106,7 @@ final class ClipboardCaptureCoordinator {
|
||||
semanticRanking.analyze(newest)
|
||||
}
|
||||
forcesNextSample = true
|
||||
autoTriggerOOBESkillIfNeeded()
|
||||
// Delay the system pasteboard read until the first poll tick. A
|
||||
// Universal Clipboard fetch or paste alert during the appear sequence
|
||||
// can otherwise freeze the keyboard before SwiftUI draws.
|
||||
@@ -104,12 +125,10 @@ final class ClipboardCaptureCoordinator {
|
||||
}
|
||||
|
||||
func refreshFlagsFromStore() {
|
||||
// Settings changes may hide the active suggestion, but enabling the
|
||||
// strip must wait for a new pasteboard generation.
|
||||
if !state.clipboardHistoryEnabled || !state.clipboardCandidateBarEnabled {
|
||||
endCurrentSuggestion()
|
||||
}
|
||||
// Paste is available whenever clipboard history is on; only turning
|
||||
// history off hides it (the old candidate-bar toggle is gone).
|
||||
if !state.clipboardHistoryEnabled {
|
||||
endCurrentSuggestion()
|
||||
semanticRanking.clear()
|
||||
}
|
||||
}
|
||||
@@ -150,6 +169,11 @@ final class ClipboardCaptureCoordinator {
|
||||
}
|
||||
|
||||
func dismissSuggestion() {
|
||||
// Dismissing from either surface must suppress both the paste capsule
|
||||
// AND the AI skill top bar for this clipboard generation. The typing
|
||||
// surface's X only routes here, so set the skill marker centrally
|
||||
// rather than relying on the AI surface's own dismiss handler.
|
||||
state.dismissedClipboardSkillEntryID = history.newestEntry?.id
|
||||
endCurrentSuggestion()
|
||||
}
|
||||
|
||||
@@ -186,6 +210,7 @@ final class ClipboardCaptureCoordinator {
|
||||
let timer = Timer(timeInterval: Self.pollInterval, repeats: true) { [weak self] _ in
|
||||
Task { @MainActor in
|
||||
self?.captureIfNeeded()
|
||||
self?.autoTriggerOOBESkillIfNeeded()
|
||||
}
|
||||
}
|
||||
timer.tolerance = Self.pollInterval / 4
|
||||
@@ -342,8 +367,9 @@ final class ClipboardCaptureCoordinator {
|
||||
}
|
||||
|
||||
private func updateSuggestion(with entry: ClipboardHistoryEntry, changeCount: Int) {
|
||||
// The one-tap Paste capsule is always on (its old opt-in toggle was
|
||||
// removed), so this only depends on clipboard history + a secure field.
|
||||
guard state.canShowClipboardEntry,
|
||||
state.clipboardCandidateBarEnabled,
|
||||
changeCount != secureFieldSuppressedChangeCount
|
||||
else {
|
||||
clearSuggestion()
|
||||
@@ -372,4 +398,273 @@ final class ClipboardCaptureCoordinator {
|
||||
state.clipboardSuggestionText = nil
|
||||
state.clipboardSuggestionChangeCount = nil
|
||||
}
|
||||
|
||||
// MARK: - Auto mode
|
||||
|
||||
/// Auto mode: on every fresh semantic analysis, pick at most one automatic
|
||||
/// action for the newest clipboard and route it into the assistant surface
|
||||
/// with no tap. Surface-independent (voice / Chinese / English) and fired at
|
||||
/// most once per pasteboard generation, persisted so a reopen never repeats.
|
||||
///
|
||||
/// Precedence when several toggles apply to the same paste:
|
||||
/// email reply (a detected email should be answered, not just translated —
|
||||
/// even in another language) → translate (foreign, non-email) → generic reply.
|
||||
private func autoTriggerActionIfNeeded() {
|
||||
guard state.clipboardHistoryEnabled,
|
||||
state.aiServiceAvailable,
|
||||
state.oobePracticeSession == nil,
|
||||
isKeyboardVisible,
|
||||
isRestingForAutoReply else {
|
||||
return
|
||||
}
|
||||
guard state.clipboardAutoModeEnabled
|
||||
|| state.clipboardAutoTranslateEnabled
|
||||
|| state.clipboardAutoEmailReplyEnabled else {
|
||||
return
|
||||
}
|
||||
guard let newest = history.newestEntry,
|
||||
let changeCount = newest.changeCount else {
|
||||
return
|
||||
}
|
||||
// One auto action per copy, even across keyboard close/reopen.
|
||||
guard history.lastAutoRepliedChangeCount != changeCount else { return }
|
||||
guard AIHintPool.isClipboardSkillWindowActive(
|
||||
clipboardHistoryEnabled: state.clipboardHistoryEnabled,
|
||||
newestClipboard: newest
|
||||
) else {
|
||||
return
|
||||
}
|
||||
guard let snapshot = semanticRanking.snapshot,
|
||||
snapshot.entryID == newest.id else {
|
||||
return
|
||||
}
|
||||
let catalog = state.clipboardSkillCatalog
|
||||
|
||||
// 1. Auto-draft a single, email-formatted reply for a detected email.
|
||||
// Checked before translate so a foreign-language email is answered
|
||||
// rather than merely translated. Staged like auto-translate: shown on
|
||||
// the keyboard, inserted only on the glass button.
|
||||
if state.clipboardAutoEmailReplyEnabled,
|
||||
ClipboardEmailDetector.isEmail(newest.text) {
|
||||
fireAutoAction(
|
||||
AIClipboardSkillCatalog.emailReplySkill,
|
||||
scene: nil,
|
||||
changeCount: changeCount,
|
||||
readOnlyResult: true,
|
||||
insertLabelKey: "keyboard.assistant.insertReply"
|
||||
)
|
||||
return
|
||||
}
|
||||
// 2. Auto-translate a non-system-language, non-email paste (staged).
|
||||
if state.clipboardAutoTranslateEnabled,
|
||||
ClipboardSkillSemanticRanker.isForeignLanguage(snapshot.analysis),
|
||||
let translate = catalog.first(where: { $0.id == AIClipboardSkillCatalog.translateID }) {
|
||||
fireAutoAction(translate, scene: nil, changeCount: changeCount, readOnlyResult: true)
|
||||
return
|
||||
}
|
||||
// 3. Auto-reply for any interpersonal message worth answering — a task,
|
||||
// question, invitation, complaint, follow-up, or an explicit "please
|
||||
// reply". Gated on reply intent (not strict #1 ranking) so a message
|
||||
// that also scores Events / Todos / Summary still auto-replies. Bare
|
||||
// links / phones / foreign text keep their own actions.
|
||||
if state.clipboardAutoModeEnabled,
|
||||
ClipboardSkillSemanticRanker.isAutoReplyEligible(
|
||||
sourceText: newest.text,
|
||||
analysis: snapshot.analysis
|
||||
),
|
||||
let reply = catalog.first(where: { $0.id == AIClipboardSkillCatalog.replyID }) {
|
||||
fireAutoAction(
|
||||
reply,
|
||||
scene: AIClipboardReplyScene.resolve(
|
||||
from: snapshot.analysis,
|
||||
sourceText: newest.text
|
||||
),
|
||||
changeCount: changeCount,
|
||||
readOnlyResult: false
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Submit the chosen auto action, remembering the keyboard to return to and
|
||||
/// taking over the assistant surface so the result is visible.
|
||||
private func fireAutoAction(
|
||||
_ skill: AIClipboardSkill,
|
||||
scene: AIClipboardReplyScene?,
|
||||
changeCount: Int,
|
||||
readOnlyResult: Bool,
|
||||
insertLabelKey: String = "keyboard.assistant.insertTranslation"
|
||||
) {
|
||||
// Mark handled before submitting so re-entrant publishes cannot double-fire.
|
||||
history.lastAutoRepliedChangeCount = changeCount
|
||||
// Remember the keyboard to return to once the result is used or closed:
|
||||
// the surface the user is actively on, or — when the keyboard opened
|
||||
// straight onto voice — the one they last left, so finishing never
|
||||
// strands them on voice input. nil means "already where we'd land".
|
||||
let returnSurface: KeyboardState.Surface = state.surface != .voice
|
||||
? state.surface
|
||||
: TypingInputConfiguration.lastLeftSurface()
|
||||
state.autoReplyReturnSurface = returnSurface == .voice ? nil : returnSurface
|
||||
// Staged auto results (translate / email reply) wait for a glass-button
|
||||
// tap; generic replies use their own variant surface.
|
||||
state.autoResultReadOnly = readOnlyResult
|
||||
state.autoResultInsertLabelKey = insertLabelKey
|
||||
// The result UI only renders on the assistant surface, so take it over.
|
||||
if state.surface != .voice {
|
||||
state.setSurface(.voice)
|
||||
}
|
||||
state.submitAIClipboardSkill(skill, scene)
|
||||
}
|
||||
|
||||
// MARK: - Auto-reply guide (one-time nudge)
|
||||
|
||||
/// Persisted "the user has already responded to the nudge once" flag. Stored
|
||||
/// in the extension's own defaults; the guide is a keyboard-local affordance.
|
||||
private static let autoReplyGuideTriedKey = "keyboard.assistant.autoReplyGuidanceTried"
|
||||
private static var autoReplyGuideTried: Bool {
|
||||
get { UserDefaults.standard.bool(forKey: autoReplyGuideTriedKey) }
|
||||
set { UserDefaults.standard.set(newValue, forKey: autoReplyGuideTriedKey) }
|
||||
}
|
||||
/// Only auto-present once per pasteboard generation, so dismissing the layer
|
||||
/// does not immediately resurface it for the same copy.
|
||||
private var lastGuidedChangeCount: Int?
|
||||
|
||||
/// For users who never turned auto mode on: when a freshly copied message
|
||||
/// reads as replyable, raise the clean full-keyboard guide instead of firing
|
||||
/// a reply. Shown on every replyable copy until the user acts on it once.
|
||||
private func presentAutoReplyGuideIfNeeded() {
|
||||
guard !state.clipboardAutoModeEnabled,
|
||||
!Self.autoReplyGuideTried,
|
||||
state.clipboardHistoryEnabled,
|
||||
state.aiServiceAvailable,
|
||||
state.oobePracticeSession == nil,
|
||||
state.canShowClipboardEntry,
|
||||
state.clipboardOverlay == .none,
|
||||
isKeyboardVisible,
|
||||
isRestingForAutoReply else {
|
||||
return
|
||||
}
|
||||
guard let newest = history.newestEntry,
|
||||
let changeCount = newest.changeCount,
|
||||
lastGuidedChangeCount != changeCount else {
|
||||
return
|
||||
}
|
||||
guard AIHintPool.isClipboardSkillWindowActive(
|
||||
clipboardHistoryEnabled: state.clipboardHistoryEnabled,
|
||||
newestClipboard: newest
|
||||
) else {
|
||||
return
|
||||
}
|
||||
guard let snapshot = semanticRanking.snapshot,
|
||||
snapshot.entryID == newest.id else {
|
||||
return
|
||||
}
|
||||
let recommended = ClipboardSkillSemanticRanker.recommended(
|
||||
skills: state.clipboardSkillCatalog,
|
||||
sourceText: newest.text,
|
||||
analysis: snapshot.analysis,
|
||||
uiLanguage: state.uiLanguage,
|
||||
limit: 5
|
||||
)
|
||||
guard recommended.first?.id == AIClipboardSkillCatalog.replyID else { return }
|
||||
lastGuidedChangeCount = changeCount
|
||||
state.clipboardOverlay = .autoReplyGuide
|
||||
}
|
||||
|
||||
/// The guide's primary action: opt into auto mode (persisted so it sticks),
|
||||
/// close the layer, and draft the reply for the copy that prompted it — the
|
||||
/// same submission every future auto reply now makes on its own.
|
||||
func confirmAutoReplyGuide() {
|
||||
Self.autoReplyGuideTried = true
|
||||
AppGroupStore().setClipboardAutoModeEnabled(true)
|
||||
state.clipboardAutoModeEnabled = true
|
||||
state.clipboardOverlay = .none
|
||||
guard let newest = history.newestEntry,
|
||||
let snapshot = semanticRanking.snapshot,
|
||||
snapshot.entryID == newest.id,
|
||||
let reply = state.clipboardSkillCatalog.first(where: {
|
||||
$0.id == AIClipboardSkillCatalog.replyID
|
||||
}) else {
|
||||
return
|
||||
}
|
||||
let scene = AIClipboardReplyScene.resolve(
|
||||
from: snapshot.analysis,
|
||||
sourceText: newest.text
|
||||
)
|
||||
fireAutoAction(
|
||||
reply,
|
||||
scene: scene,
|
||||
changeCount: newest.changeCount ?? history.lastObservedChangeCount,
|
||||
readOnlyResult: false
|
||||
)
|
||||
}
|
||||
|
||||
/// OOBE reply / translate practice: once the host has seeded the demo
|
||||
/// message, run the skill automatically — the same submission the chip makes
|
||||
/// — so the user experiences the auto skill before any setting is turned on.
|
||||
/// Runs off the poll tick (OOBE material is not semantic-ranked), and fires
|
||||
/// at most once per practice feature within the host session. The translate
|
||||
/// result is staged (glass Insert button) exactly like the live auto flow.
|
||||
private func autoTriggerOOBESkillIfNeeded() {
|
||||
guard let session = state.oobePracticeSession,
|
||||
state.aiServiceAvailable,
|
||||
isKeyboardVisible,
|
||||
isRestingForAutoReply else {
|
||||
return
|
||||
}
|
||||
let feature = session.expectedFeature
|
||||
let skillID: String
|
||||
switch feature {
|
||||
case .clipboardReply:
|
||||
skillID = AIClipboardSkillCatalog.replyID
|
||||
case .clipboardTranslate:
|
||||
skillID = AIClipboardSkillCatalog.translateID
|
||||
case .voiceInput, .askAI:
|
||||
return
|
||||
}
|
||||
// The host keeps one session id across steps, so gate per feature.
|
||||
if oobeAutoFiredSessionID != session.sessionID {
|
||||
oobeAutoFiredSessionID = session.sessionID
|
||||
oobeAutoFiredFeatures = []
|
||||
}
|
||||
guard !oobeAutoFiredFeatures.contains(feature) else { return }
|
||||
guard KeyboardSetupBridge.oobeClipboardMaterial(
|
||||
sessionID: session.sessionID
|
||||
) != nil else {
|
||||
return
|
||||
}
|
||||
guard let skill = state.clipboardSkillCatalog.first(where: {
|
||||
$0.id == skillID
|
||||
}) else {
|
||||
return
|
||||
}
|
||||
// Mark handled before submitting so a re-entrant tick cannot double-fire.
|
||||
oobeAutoFiredFeatures.insert(feature)
|
||||
// Stage the translation like the live auto-translate: shown on the
|
||||
// keyboard, inserted only when the user taps the glass button.
|
||||
if feature == .clipboardTranslate {
|
||||
state.autoResultReadOnly = true
|
||||
state.autoResultInsertLabelKey = "keyboard.assistant.insertTranslation"
|
||||
}
|
||||
// The result UI only renders on the assistant surface, so take it over.
|
||||
if state.surface != .voice {
|
||||
state.setSurface(.voice)
|
||||
}
|
||||
state.submitAIClipboardSkill(skill, nil)
|
||||
}
|
||||
|
||||
/// The voice pipeline must be idle before auto mode takes over the surface.
|
||||
private var isRestingForAutoReply: Bool {
|
||||
guard !state.aiSession.isBusy,
|
||||
!state.aiSession.canInsert,
|
||||
!state.aiSession.canSelectReplyVariant,
|
||||
!state.editSession.isActive else {
|
||||
return false
|
||||
}
|
||||
switch state.phase {
|
||||
case .idle, .error, .denied:
|
||||
return true
|
||||
case .requestingPermissions, .recording, .processing:
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user