rockyandClaude Opus 5
0023741455
Squashes the working tree that had accumulated on cloud-agent. Two strands of work, kept in one commit at the maintainer's request. Crash and stability (this session): - Subscribe to MetricKit so crash, hang, CPU, and disk-write diagnostics are recorded on device. Keyboard-extension memory kills are the most likely way a user loses the keyboard mid-sentence and iOS does not classify them as crashes, so they never reach App Store Connect. - Upload those reports to POST /v1/analytics/diagnostics from internal builds only (local Debug and TestFlight), reusing the analytics installation identifier and skipping entirely when analytics are off. App Store builds keep everything on device. - Shed the Chinese engine, English lexicon, and in-flight pipeline work at the 40 MiB and 48 MiB budget thresholds instead of waiting for a system memory warning that jetsam often preempts. - Install audio taps with the input node's live format through one shared adaptive resampler, so an audio-route change mid-recording no longer raises an uncatchable format-mismatch exception. Covers the dictation preview and the Mac recorder, which still used the pattern the Flow capture path had already moved away from. - Harden the librime bridge: never pass a nil string across a nonnull boundary when a user dictionary holds malformed UTF-8, null-check the engine API vector, and distrust a candidate count without its array. - Guard the expanded candidate grid against a list that shrinks while the collection view is still laying out against the previous count. - Resolve localized text without the App Group so the screen shown when that container is missing renders its message instead of crashing on launch, which is the one situation it exists to handle. - Gate CI on the App Group and Keychain wiring that the keyboard extension hard-depends on, with a self-test proving the check fails. Clipboard, reply, and semantics (pre-existing working tree): - Unify the reply center, refresh keyboard AI features, and add native event and reminder exporters. - Expand clipboard semantic routing with authored corpora and a product-blind holdout, plus the scripts that build and validate them. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1380 lines
58 KiB
Swift
1380 lines
58 KiB
Swift
// KeyboardViewController.swift
|
||
// OSGKeyboard · Keyboard Extension
|
||
//
|
||
// Principal class for the Custom Keyboard Extension. Hosts a single
|
||
// SwiftUI tree (`KeyboardSurfaceRoot`) and wires Flow voice / AI input:
|
||
//
|
||
// host app Flow session ──► App Group transcript ──► insertText
|
||
//
|
||
// Design notes:
|
||
// • The class is `@MainActor` — every UI mutation and `textDocumentProxy`
|
||
// call must happen on main, and Swift 6 strict concurrency forces this.
|
||
// • State is a single `State` ObservableObject; SwiftUI observes it via
|
||
// `@ObservedObject` so we never re-create the hosting root on each tick.
|
||
// • `phase` is a real stored property (no derivation) — the previous
|
||
// "derive from recordStream" shim locked out every press after the first.
|
||
|
||
import Combine
|
||
import OSGKeyboardShared
|
||
import SwiftUI
|
||
import UIKit
|
||
|
||
private final class KeyboardHostingController: UIHostingController<KeyboardSurfaceRoot> {
|
||
override var preferredScreenEdgesDeferringSystemGestures: UIRectEdge {
|
||
[.left, .right]
|
||
}
|
||
|
||
override func viewDidAppear(_ animated: Bool) {
|
||
super.viewDidAppear(animated)
|
||
setNeedsUpdateOfScreenEdgesDeferringSystemGestures()
|
||
}
|
||
}
|
||
|
||
@objc(KeyboardViewController)
|
||
@MainActor
|
||
public final class KeyboardViewController: UIInputViewController {
|
||
public typealias State = KeyboardState
|
||
|
||
private let state = State()
|
||
/// Created on first typing use so the default voice surface never pays
|
||
/// for `TypingSessionController` / engine factories at KVC init.
|
||
private var typingSessionStorage: TypingSessionController?
|
||
private var typingSession: TypingSessionController {
|
||
if let typingSessionStorage { return typingSessionStorage }
|
||
let created = TypingSessionController()
|
||
created.systemLexicon = UIKitEnglishSystemLexicon()
|
||
typingSessionStorage = created
|
||
return created
|
||
}
|
||
private let persistor = AppGroupPersistor()
|
||
/// Uptime of the most recent key-driven document change. Memory relief uses
|
||
/// it to tell "the user is mid-sentence" from "the typing surface is merely
|
||
/// on screen" — see `isMidTypingBurst`.
|
||
private var lastTypingActivityUptime: TimeInterval = -1
|
||
|
||
private var hosting: UIHostingController<KeyboardSurfaceRoot>?
|
||
private var keyboardHeightConstraint: NSLayoutConstraint?
|
||
/// The system owns the input view's height during the slide-in via a
|
||
/// required `UIView-Encapsulated-Layout-Height` constraint, which it walks
|
||
/// from the full screen height down to the keyboard slot. Our own
|
||
/// constraint only has to hold `target` and stay out of that transition.
|
||
private enum HeightPresentationPhase {
|
||
case idle
|
||
case presented
|
||
}
|
||
private var heightPhase: HeightPresentationPhase = .idle
|
||
/// Last logged layout snapshot, so `viewDidLayoutSubviews` only reports
|
||
/// changes instead of every pass.
|
||
private var lastLoggedLayoutSnapshot: String?
|
||
private var cancellables = Set<AnyCancellable>()
|
||
/// Coalesces host-document refreshes after mutations issued by this keyboard.
|
||
private var assistantFieldActionRefreshTask: Task<Void, Never>?
|
||
/// One random ID per keyboard presentation. The repository splits this ID
|
||
/// into independent UTC-day fragments when a presentation crosses midnight.
|
||
private var keyboardUsageSessionID = UUID()
|
||
/// UIKit can reuse this controller across host apps. Keep document-scoped
|
||
/// candidate state separate from the heavy typing-engine lifetime.
|
||
private var isKeyboardPresentationActive = false
|
||
private var isTypingSurfaceActive = false
|
||
private var typingDocumentPresentationID: UInt64?
|
||
|
||
private var memoryTelemetryContext: String {
|
||
let language = typingSessionStorage?.language.rawValue ?? "-"
|
||
return "surface=\(state.surface.rawValue) language=\(language) "
|
||
+ "fullAccess=\(hasFullAccess ? 1 : 0) "
|
||
+ "clipboard=\(state.clipboardHistoryEnabled ? 1 : 0)"
|
||
}
|
||
|
||
private func recordMemory(_ stage: String, details: String? = nil) {
|
||
KeyboardExtensionMemoryTelemetry.updateContext(memoryTelemetryContext)
|
||
KeyboardExtensionMemoryTelemetry.record(stage, details: details)
|
||
}
|
||
|
||
private var editHintScheduler: EditHintScheduler!
|
||
private var textInserter: KeyboardTextInserter!
|
||
private var flowCoordinator: KeyboardFlowCoordinator!
|
||
private var lastInputEditCoordinator: LastInputEditCoordinator!
|
||
private var aiKeyboardCoordinator: AIKeyboardCoordinator!
|
||
private var clipboardCapture: ClipboardCaptureCoordinator!
|
||
private var configSync: KeyboardConfigSync!
|
||
/// iPad-scale keys require both an iPad host and regular horizontal space.
|
||
/// This keeps compact iPad multitasking on phone metrics and prevents wide
|
||
/// iPhones from being mistaken for iPads.
|
||
private var isIPadLayout: Bool {
|
||
KeyboardChromeLayout.usesIPadMetrics(
|
||
isPad: UIDevice.current.userInterfaceIdiom == .pad,
|
||
hasRegularWidth: traitCollection.horizontalSizeClass == .regular
|
||
)
|
||
}
|
||
|
||
/// Width the layout should be sized against. `view.bounds` is empty before
|
||
/// the first layout pass, so fall back to the screen the keyboard is on.
|
||
private var currentLayoutWidth: CGFloat {
|
||
let width = view.bounds.width
|
||
guard width > 0 else {
|
||
return (view.window?.windowScene?.screen ?? UIScreen.main).bounds.width
|
||
}
|
||
return width
|
||
}
|
||
|
||
private var targetKeyboardHeight: CGFloat {
|
||
KeyboardSurfaceRoot.height(
|
||
for: state.surface,
|
||
isIPad: state.usesIPadLayoutMetrics,
|
||
width: state.layoutWidth
|
||
)
|
||
}
|
||
|
||
// MARK: - Init
|
||
|
||
public override init(nibName nibNameOrNil: String?, bundle nibBundleOrNil: Bundle?) {
|
||
KeyboardExtensionMemoryTelemetry.begin(
|
||
context: "surface=uninitialized language=- fullAccess=- clipboard=-"
|
||
)
|
||
OSGDiag.log("KVC.init(nib) begin \(OSGDiag.memoryTag())", category: "boot")
|
||
super.init(nibName: nibNameOrNil, bundle: nibBundleOrNil)
|
||
recordMemory("KVC.init.done")
|
||
OSGDiag.log("KVC.init(nib) done \(OSGDiag.memoryTag())", category: "boot")
|
||
}
|
||
|
||
public required init?(coder: NSCoder) {
|
||
KeyboardExtensionMemoryTelemetry.begin(
|
||
context: "surface=uninitialized language=- fullAccess=- clipboard=-"
|
||
)
|
||
OSGDiag.log("KVC.init(coder) begin \(OSGDiag.memoryTag())", category: "boot")
|
||
super.init(coder: coder)
|
||
recordMemory("KVC.init.done")
|
||
OSGDiag.log("KVC.init(coder) done \(OSGDiag.memoryTag())", category: "boot")
|
||
}
|
||
|
||
deinit {
|
||
// Intentionally NSLog-only: deinit is nonisolated.
|
||
NSLog("%@", "[OSGDiag/boot] KVC.deinit")
|
||
}
|
||
|
||
// MARK: - Lifecycle
|
||
|
||
public override func viewDidLoad() {
|
||
super.viewDidLoad()
|
||
state.showsSystemGlobeKey = UIDevice.current.userInterfaceIdiom == .pad
|
||
// Voice-first keyboard — hide the misleading "English" subtitle in Settings.
|
||
primaryLanguage = "mis"
|
||
let preferred = TypingInputConfiguration.preferredSurfaceOnOpen()
|
||
recordMemory(
|
||
"KVC.viewDidLoad.begin",
|
||
details: "preferredSurface=\(preferred.rawValue)"
|
||
)
|
||
KeyboardExtensionMemoryTelemetry.startBootSampling()
|
||
OSGDiag.log(
|
||
"KVC.viewDidLoad begin preferredSurface=\(preferred.rawValue) "
|
||
+ "fullAccess=\(hasFullAccess) \(OSGDiag.memoryTag())",
|
||
category: "boot"
|
||
)
|
||
// Deliberately NO CustomLanguageModelManager prewarm here: the
|
||
// extension never runs ASR (the host app owns the microphone and
|
||
// the SpeechAnalyzer pipeline), and compiling/caching an LM inside
|
||
// the keyboard's ~60 MB jetsam budget risks the system killing the
|
||
// keyboard outright. The host app prewarms it on session start.
|
||
setNeedsUpdateOfScreenEdgesDeferringSystemGestures()
|
||
// Establish layout dependencies before applying the preferred surface.
|
||
// `applySurface` updates height and UIKit may lay out synchronously.
|
||
refreshLayoutMode()
|
||
installKeyboardHeight()
|
||
configureDictationBehavior()
|
||
installServices()
|
||
recordMemory("KVC.viewDidLoad.afterInstallServices")
|
||
OSGDiag.log("KVC.viewDidLoad after installServices \(OSGDiag.memoryTag())", category: "boot")
|
||
// Apply open preference before mounting SwiftUI so the first frame is
|
||
// already voice or typing — avoids a visible surface flash.
|
||
applyPreferredSurfaceOnOpen()
|
||
recordMemory("KVC.viewDidLoad.afterPreferredSurface")
|
||
OSGDiag.log("KVC.viewDidLoad after preferredSurface surface=\(state.surface.rawValue)", category: "boot")
|
||
installTypingContextProviders()
|
||
installStateActions()
|
||
installSurfaceObservers()
|
||
installSwiftUI()
|
||
recordMemory("KVC.viewDidLoad.afterInstallSwiftUI")
|
||
OSGDiag.log("KVC.viewDidLoad after installSwiftUI \(OSGDiag.memoryTag())", category: "boot")
|
||
let configLoadResult = configSync.loadPersistedConfig()
|
||
refreshMultipleReplyVariantsSetting()
|
||
recordMemory(
|
||
"KVC.viewDidLoad.afterConfigLoad",
|
||
details: "result=\(configLoadResult)"
|
||
)
|
||
configSync.installDarwinObservers()
|
||
flowCoordinator.refreshSessionState()
|
||
// Installed last: the handler tears down surfaces, so it must not be
|
||
// reachable until the whole controller is wired up.
|
||
KeyboardExtensionMemoryTelemetry.reliefHandler = { [weak self] level in
|
||
self?.shedHeavyResources(level: level, reason: "budget") ?? false
|
||
}
|
||
recordMemory(
|
||
"KVC.viewDidLoad.done",
|
||
details: "sessionActive=\(FlowSessionBridge.isSessionActive() ? 1 : 0) "
|
||
+ "hostReady=\(FlowSessionBridge.isHostReady() ? 1 : 0)"
|
||
)
|
||
OSGDiag.log(
|
||
"KVC.viewDidLoad done surface=\(state.surface.rawValue) "
|
||
+ "sessionActive=\(FlowSessionBridge.isSessionActive()) "
|
||
+ "hostReady=\(FlowSessionBridge.isHostReady()) \(OSGDiag.memoryTag())",
|
||
category: "boot"
|
||
)
|
||
}
|
||
|
||
public override func viewWillDisappear(_ animated: Bool) {
|
||
super.viewWillDisappear(animated)
|
||
isKeyboardPresentationActive = false
|
||
typingDocumentPresentationID = nil
|
||
typingSessionStorage?.endDocumentPresentation()
|
||
AnalyticsExtensionService.shared.keyboardWillDisappear()
|
||
assistantFieldActionRefreshTask?.cancel()
|
||
assistantFieldActionRefreshTask = nil
|
||
clipboardCapture?.keyboardWillDisappear()
|
||
// Presentation-scoped hints must never survive a reused extension
|
||
// controller, including an active Flow handoff.
|
||
editHintScheduler?.invalidate()
|
||
OSGDiag.log(
|
||
"KVC.viewWillDisappear surface=\(state.surface.rawValue) "
|
||
+ "preserve=\(flowCoordinator.preservesLifecycleOnDisappear) \(OSGDiag.memoryTag())",
|
||
category: "boot"
|
||
)
|
||
recordMemory(
|
||
"KVC.viewWillDisappear",
|
||
details: "preserve=\(flowCoordinator.preservesLifecycleOnDisappear ? 1 : 0)"
|
||
)
|
||
heightPhase = .idle
|
||
// A hidden keyboard cannot grow; stop waking the process to sample it.
|
||
KeyboardExtensionMemoryTelemetry.stopSampling()
|
||
flowCoordinator.stopSessionMonitor()
|
||
// Remember what the user left on, then pre-position a reused
|
||
// extension instance for the next open policy (no first-frame jump).
|
||
let preserve = flowCoordinator.preservesLifecycleOnDisappear
|
||
TypingInputConfiguration.persistLastSurface(
|
||
preserve ? .voice : state.surface
|
||
)
|
||
// Remember pinyin/English with the surface so "remember last" restores both.
|
||
if state.surface == .typing {
|
||
TypingInputConfiguration.persistLastTypingLanguage(typingSession.language)
|
||
}
|
||
if state.aiSession.isActive {
|
||
// Unified assistant conversations never survive a keyboard presentation.
|
||
aiKeyboardCoordinator.leave()
|
||
}
|
||
if !preserve {
|
||
prepareSurfaceForNextPresentation()
|
||
}
|
||
if state.surface != .typing {
|
||
deactivateTypingSurface()
|
||
}
|
||
if preserve {
|
||
return
|
||
}
|
||
ExtensionScreenWakeLock.releaseAll()
|
||
flowCoordinator.cancelPipelineUnlessAwaitingResult()
|
||
}
|
||
|
||
public override func viewWillAppear(_ animated: Bool) {
|
||
super.viewWillAppear(animated)
|
||
isKeyboardPresentationActive = true
|
||
OSGDiag.log(
|
||
"KVC.viewWillAppear begin surface=\(state.surface.rawValue) "
|
||
+ "fullAccess=\(hasFullAccess) \(OSGDiag.memoryTag())",
|
||
category: "boot"
|
||
)
|
||
recordMemory("KVC.viewWillAppear.begin")
|
||
setNeedsUpdateOfScreenEdgesDeferringSystemGestures()
|
||
configureDictationBehavior()
|
||
state.hasFullAccess = hasFullAccess
|
||
KeyboardSetupBridge.markExtensionAppearance(hasFullAccess: hasFullAccess)
|
||
// Refresh only Flow/config state; edit targets come from verified OSG insertions.
|
||
flowCoordinator.refreshSessionState()
|
||
flowCoordinator.startSessionMonitor()
|
||
configSync.syncOnboardingStateFromAppGroup()
|
||
configSync.refreshConfigFromAppGroup()
|
||
refreshMultipleReplyVariantsSetting()
|
||
clipboardCapture.refreshFlagsFromStore()
|
||
// Settings may have changed while the extension stayed alive.
|
||
applyPreferredSurfaceOnOpen()
|
||
// Re-warm Taptic after host app switches: SwiftUI `onAppear` often
|
||
// skips when the extension process is reused, leaving generators cold.
|
||
KeyboardHapticFeedback.prepare()
|
||
recordMemory("KVC.viewWillAppear.afterHaptics")
|
||
if state.surface == .typing {
|
||
OSGDiag.log("KVC.viewWillAppear enterTypingMode", category: "boot")
|
||
activateTypingPresentationIfNeeded()
|
||
recordMemory("KVC.viewWillAppear.afterTypingEnter")
|
||
}
|
||
clipboardCapture.keyboardDidAppear()
|
||
recordMemory("KVC.viewWillAppear.afterClipboard")
|
||
// Re-presented keyboards reuse the same process, so resume pressure
|
||
// polling here. No-ops while the first-launch boot burst is still
|
||
// running, so the startup samples are never cut short.
|
||
KeyboardExtensionMemoryTelemetry.startSustainedSamplingIfIdle()
|
||
recordMemory("KVC.viewWillAppear.done")
|
||
OSGDiag.log(
|
||
"KVC.viewWillAppear done surface=\(state.surface.rawValue) \(OSGDiag.memoryTag())",
|
||
category: "boot"
|
||
)
|
||
}
|
||
|
||
public override func viewIsAppearing(_ animated: Bool) {
|
||
super.viewIsAppearing(animated)
|
||
// One constant, every time: the previous "prime at target − system
|
||
// encapsulated height" trick read the pre-presentation full-screen
|
||
// height (874 pt on an iPhone), clamped to 0, and could not win against
|
||
// the system's required constraint anyway.
|
||
lockPresentedKeyboardHeight()
|
||
OSGDiag.log(
|
||
"KVC.viewIsAppearing phase=\(heightPhaseLog) "
|
||
+ "height=\(keyboardHeightConstraint?.constant ?? -1) "
|
||
+ "\(OSGDiag.memoryTag())",
|
||
category: "boot"
|
||
)
|
||
logHeightConstraints(tag: "viewIsAppearing")
|
||
}
|
||
|
||
public override func viewDidAppear(_ animated: Bool) {
|
||
super.viewDidAppear(animated)
|
||
keyboardUsageSessionID = UUID()
|
||
AnalyticsExtensionService.shared.recordPresentation(
|
||
hasFullAccess: hasFullAccess
|
||
)
|
||
OSGDiag.log(
|
||
"KVC.viewDidAppear begin surface=\(state.surface.rawValue) \(OSGDiag.memoryTag())",
|
||
category: "boot"
|
||
)
|
||
recordMemory("KVC.viewDidAppear.begin")
|
||
disableSystemGestureDelays()
|
||
heightPhase = .presented
|
||
lockPresentedKeyboardHeight()
|
||
refreshReturnKeyRole()
|
||
scheduleTypingDocumentSynchronization()
|
||
// Presentation math is locked before arming the PiP handoff.
|
||
DispatchQueue.main.async { [weak self] in
|
||
guard let self, self.heightPhase == .presented else { return }
|
||
self.flowCoordinator.ensurePiPReadyOnKeyboardOpen()
|
||
}
|
||
#if DEBUG
|
||
WhatsNewDemoDriver.resetForNewPresentation()
|
||
WhatsNewDemoDriver.startIfNeeded(
|
||
state: state,
|
||
host: WhatsNewDemoDriver.HostHooks(
|
||
insertText: { [weak self] text in
|
||
self?.insertTextIntoDocument(text, source: .debugDemo)
|
||
},
|
||
deleteBackward: { [weak self] in
|
||
self?.deleteBackwardFromDocument()
|
||
},
|
||
contextBeforeInput: { [weak self] in
|
||
self?.textDocumentProxy.documentContextBeforeInput
|
||
},
|
||
performReturn: { [weak self] in
|
||
self?.performDocumentReturn()
|
||
}
|
||
)
|
||
)
|
||
#endif
|
||
OSGDiag.log(
|
||
"KVC.viewDidAppear done height=\(targetKeyboardHeight) \(OSGDiag.memoryTag())",
|
||
category: "boot"
|
||
)
|
||
recordMemory("KVC.viewDidAppear.done")
|
||
}
|
||
|
||
public override func textDidChange(_ textInput: UITextInput?) {
|
||
super.textDidChange(textInput)
|
||
refreshReturnKeyRole()
|
||
synchronizeTypingDocumentContext()
|
||
textInserter?.refreshEditingAvailability()
|
||
lastInputEditCoordinator?.refreshContext()
|
||
}
|
||
|
||
public override func selectionDidChange(_ textInput: UITextInput?) {
|
||
super.selectionDidChange(textInput)
|
||
refreshReturnKeyRole()
|
||
synchronizeTypingDocumentContext(caretMoved: true)
|
||
textInserter?.refreshEditingAvailability()
|
||
lastInputEditCoordinator?.refreshContext()
|
||
}
|
||
|
||
public override func traitCollectionDidChange(_ previousTraitCollection: UITraitCollection?) {
|
||
super.traitCollectionDidChange(previousTraitCollection)
|
||
guard previousTraitCollection?.horizontalSizeClass != traitCollection.horizontalSizeClass else {
|
||
return
|
||
}
|
||
refreshLayoutMode()
|
||
}
|
||
|
||
public override var preferredScreenEdgesDeferringSystemGestures: UIRectEdge {
|
||
[.left, .right]
|
||
}
|
||
|
||
public override var childForScreenEdgesDeferringSystemGestures: UIViewController? {
|
||
hosting
|
||
}
|
||
|
||
public override func didReceiveMemoryWarning() {
|
||
super.didReceiveMemoryWarning()
|
||
recordMemory("KVC.didReceiveMemoryWarning")
|
||
OSGDiag.log(
|
||
"KVC.didReceiveMemoryWarning surface=\(state.surface.rawValue) \(OSGDiag.memoryTag())",
|
||
category: "boot"
|
||
)
|
||
// The system signal is not an advance warning — jetsam may be moments
|
||
// away — so this path sheds even mid-composition.
|
||
_ = shedHeavyResources(level: .critical, reason: "systemWarning", force: true)
|
||
}
|
||
|
||
/// Whether the user is actively composing right now. A non-empty preedit is
|
||
/// an unfinished Pinyin syllable, and a keystroke within the last window
|
||
/// means a burst is in progress even between syllables.
|
||
private var isMidTypingBurst: Bool {
|
||
if let session = typingSessionStorage, !session.composition.preedit.isEmpty {
|
||
return true
|
||
}
|
||
guard lastTypingActivityUptime >= 0 else { return false }
|
||
return ProcessInfo.processInfo.systemUptime - lastTypingActivityUptime
|
||
< Self.typingBurstWindow
|
||
}
|
||
|
||
/// How long after a keystroke the keyboard still counts as "being typed on".
|
||
private static let typingBurstWindow: TimeInterval = 1.5
|
||
|
||
/// Releases the keyboard's heavy caches (Rime engine, English lexicon,
|
||
/// in-flight pipeline work).
|
||
///
|
||
/// Two callers: the system memory warning above, and — crucially, *earlier*
|
||
/// — `KeyboardExtensionMemoryTelemetry.reliefHandler`. Jetsam kills
|
||
/// keyboards at the ~60 MiB boundary without reliably delivering a warning
|
||
/// first, so waiting for the system signal means the keyboard simply
|
||
/// vanishes mid-sentence. Our own 40 / 48 MiB thresholds fire while there
|
||
/// is still headroom to recover.
|
||
/// Returns whether relief actually happened, so a declined attempt keeps the
|
||
/// level armed for the next poll instead of counting as a completed shed.
|
||
@discardableResult
|
||
private func shedHeavyResources(
|
||
level: KeyboardExtensionMemoryBudget.Level,
|
||
reason: String,
|
||
force: Bool = false
|
||
) -> Bool {
|
||
// Relief can be requested from the boot sampler before `installServices`
|
||
// has run; nothing below is safe to touch until the view is loaded.
|
||
guard isViewLoaded, flowCoordinator != nil else { return false }
|
||
OSGDiag.log(
|
||
"KVC.shed level=\(level.rawValue) reason=\(reason) "
|
||
+ "surface=\(state.surface.rawValue) \(OSGDiag.memoryTag())",
|
||
category: "memory"
|
||
)
|
||
flowCoordinator.cancelPipelineUnlessAwaitingResult()
|
||
guard level == .critical else {
|
||
// `.high`: release only what the user is not looking at. Yanking an
|
||
// active typing surface at 40 MiB would be more disruptive than the
|
||
// risk it avoids.
|
||
if state.surface != .typing {
|
||
deactivateTypingSurface()
|
||
}
|
||
recordMemory("KVC.shed.soft", details: "reason=\(reason)")
|
||
return true
|
||
}
|
||
guard state.surface == .typing else {
|
||
deactivateTypingSurface()
|
||
recordMemory("KVC.shed.hard", details: "reason=\(reason)")
|
||
return true
|
||
}
|
||
// Swapping the surface out from under someone mid-syllable loses the
|
||
// composition and drops them into voice mode with no explanation. A
|
||
// jetsam kill costs the same composition but reads as "the keyboard
|
||
// crashed", which is the lesser of the two for the user's trust in the
|
||
// keyboard — and the poll comes back in a second to try again.
|
||
guard force || !isMidTypingBurst else {
|
||
recordMemory("KVC.shed.deferred", details: "reason=\(reason) cause=typing")
|
||
return false
|
||
}
|
||
applySurface(.voice)
|
||
recordMemory("KVC.shed.hard", details: "reason=\(reason)")
|
||
return true
|
||
}
|
||
|
||
public override func viewDidLayoutSubviews() {
|
||
super.viewDidLayoutSubviews()
|
||
// Rotation does not change `horizontalSizeClass` on iPad (both
|
||
// orientations are regular), so this is the only callback that sees a
|
||
// portrait→landscape resize. `refreshLayoutMode` no-ops unless the
|
||
// layout bucket actually changed, so this cannot loop.
|
||
refreshLayoutMode()
|
||
enforcePresentedKeyboardHeightIfNeeded()
|
||
logLayoutSnapshotIfChanged()
|
||
}
|
||
|
||
// MARK: - Services
|
||
|
||
private func installServices() {
|
||
editHintScheduler = EditHintScheduler(state: state)
|
||
textInserter = KeyboardTextInserter(
|
||
state: state,
|
||
insertText: { [weak self] text, source in
|
||
self?.insertTextIntoDocument(text, source: source)
|
||
},
|
||
deleteBackward: { [weak self] in self?.deleteBackwardFromDocument() },
|
||
contextBeforeInput: { [weak self] in
|
||
guard let self, self.textDocumentProxy.isSecureTextEntry == false else { return nil }
|
||
return self.textDocumentProxy.documentContextBeforeInput
|
||
},
|
||
fieldContextProvider: { [weak self] in self?.captureFieldContext() },
|
||
selectedText: { [weak self] in
|
||
guard let self, self.textDocumentProxy.isSecureTextEntry == false else { return nil }
|
||
return self.textDocumentProxy.selectedText
|
||
},
|
||
scheduleAutoClearError: { [weak self] in self?.scheduleAutoClearError() },
|
||
editHintScheduler: editHintScheduler
|
||
)
|
||
|
||
configSync = KeyboardConfigSync(
|
||
state: state,
|
||
persistor: persistor,
|
||
onFlowSessionChanged: { [weak self] in
|
||
self?.flowCoordinator.refreshSessionState()
|
||
},
|
||
onConfigChanged: { [weak self] in
|
||
// Only an already-live typing session can be showing the setup
|
||
// error; never force-create one just to retry.
|
||
self?.typingSessionStorage?.reloadPersonalDictionaryTerms()
|
||
self?.typingSessionStorage?.retryPrepareAfterResourceDeployment()
|
||
self?.refreshMultipleReplyVariantsSetting()
|
||
}
|
||
)
|
||
|
||
flowCoordinator = KeyboardFlowCoordinator(
|
||
state: state,
|
||
textInserter: textInserter,
|
||
hasFullAccess: { [weak self] in self?.hasFullAccess ?? false },
|
||
wakeLockView: { [weak self] in self?.view },
|
||
openHostApp: { [weak self] path in self?.openHostApp(path: path) },
|
||
detectAndStoreAppContext: { [weak self] in self?.detectAndStoreAppContext() },
|
||
fieldContextProvider: { [weak self] in self?.captureFieldContext() },
|
||
scheduleAutoClearError: { [weak self] in self?.scheduleAutoClearError() },
|
||
refreshConfigFromAppGroup: { [weak self] in self?.configSync.refreshConfigFromAppGroup() }
|
||
)
|
||
lastInputEditCoordinator = LastInputEditCoordinator(
|
||
state: state,
|
||
textInserter: textInserter,
|
||
beginFlow: { [weak self] reference in
|
||
self?.flowCoordinator.beginEditRecording(reference: reference)
|
||
?? .rejected(.hostUnavailable)
|
||
},
|
||
stopFlow: { [weak self] in self?.flowCoordinator.stopEditRecording() },
|
||
abortFlow: { [weak self] in self?.flowCoordinator.abortEditRecording() },
|
||
acknowledge: { [weak self] outcome in
|
||
self?.flowCoordinator.acknowledgeEditResult(outcome)
|
||
},
|
||
editHintScheduler: editHintScheduler
|
||
)
|
||
flowCoordinator.onEditHostRecordingConfirmed = { [weak self] in
|
||
self?.lastInputEditCoordinator.hostRecordingConfirmed()
|
||
}
|
||
flowCoordinator.onEditResult = { [weak self] result in
|
||
self?.lastInputEditCoordinator.receive(result: result)
|
||
}
|
||
flowCoordinator.onEditFailure = { [weak self] message in
|
||
self?.lastInputEditCoordinator.fail(message)
|
||
}
|
||
aiKeyboardCoordinator = AIKeyboardCoordinator(
|
||
state: state,
|
||
flow: flowCoordinator,
|
||
insertAnswer: { [weak self] answer in
|
||
self?.textInserter.insertAIAnswer(answer) ?? false
|
||
},
|
||
performReturn: { [weak self] in
|
||
self?.performDocumentReturn()
|
||
},
|
||
captureInsertionFingerprint: { [weak self] in
|
||
self?.captureFieldContext().deliveryFingerprint
|
||
},
|
||
openWebURL: { [weak self] url in
|
||
self?.openWebURL(url)
|
||
},
|
||
callPhone: { [weak self] phoneNumber in
|
||
self?.callPhone(phoneNumber)
|
||
},
|
||
createContact: { [weak self] phoneNumber in
|
||
self?.createContact(phoneNumber)
|
||
}
|
||
)
|
||
clipboardCapture = ClipboardCaptureCoordinator(state: state)
|
||
clipboardCapture.configure(
|
||
isSecure: { [weak self] in
|
||
self?.textDocumentProxy.isSecureTextEntry ?? true
|
||
},
|
||
hasFullAccess: { [weak self] in
|
||
self?.hasFullAccess ?? false
|
||
}
|
||
)
|
||
flowCoordinator.onAIUtterancePrepared = { [weak self] utteranceID in
|
||
self?.aiKeyboardCoordinator.utterancePrepared(utteranceID)
|
||
}
|
||
flowCoordinator.onAIRecordingStarted = { [weak self] utteranceID in
|
||
self?.aiKeyboardCoordinator.recordingStarted(utteranceID)
|
||
}
|
||
flowCoordinator.onAIRecognitionStarted = { [weak self] utteranceID in
|
||
self?.aiKeyboardCoordinator.recognitionStarted(utteranceID)
|
||
}
|
||
flowCoordinator.onAIGeneratingStarted = { [weak self] utteranceID in
|
||
self?.aiKeyboardCoordinator.generatingStarted(utteranceID)
|
||
}
|
||
flowCoordinator.onAITranscript = { [weak self] transcript, utteranceID, status in
|
||
self?.aiKeyboardCoordinator.receiveTranscript(
|
||
transcript,
|
||
utteranceID: utteranceID,
|
||
status: status
|
||
)
|
||
}
|
||
flowCoordinator.onAIStreamingAnswer = { [weak self] draft, utteranceID in
|
||
self?.aiKeyboardCoordinator.receivePartialAnswer(
|
||
draft,
|
||
utteranceID: utteranceID
|
||
)
|
||
}
|
||
flowCoordinator.onAIResult = { [weak self] result in
|
||
self?.aiKeyboardCoordinator.receive(result: result)
|
||
}
|
||
flowCoordinator.onAIFailure = { [weak self] message, utteranceID in
|
||
self?.aiKeyboardCoordinator.fail(message, utteranceID: utteranceID)
|
||
}
|
||
flowCoordinator.onAICancelled = { [weak self] in
|
||
self?.state.aiSession.cancelCurrentWork()
|
||
}
|
||
_ = textInserter.recoverPendingEditTransactionIfNeeded()
|
||
}
|
||
|
||
// MARK: - Wiring
|
||
|
||
private func installStateActions() {
|
||
state.beginRecording = { [weak self] in self?.flowCoordinator.pressBegan() }
|
||
state.endRecording = { [weak self] in self?.flowCoordinator.pressEnded() }
|
||
state.tapMic = { [weak self] in self?.flowCoordinator.toggleRecording() }
|
||
state.setMicTouchActive = { [weak self] active in
|
||
self?.flowCoordinator.setMicTouchActive(active)
|
||
}
|
||
state.cancelVoiceInput = { [weak self] in
|
||
self?.flowCoordinator.cancelCurrentDictation()
|
||
}
|
||
state.beginEditLastInput = { [weak self] in
|
||
self?.lastInputEditCoordinator.begin()
|
||
}
|
||
state.stopEditListening = { [weak self] in
|
||
self?.lastInputEditCoordinator.stopListening()
|
||
}
|
||
state.confirmEditResult = { [weak self] in
|
||
self?.lastInputEditCoordinator.confirm()
|
||
}
|
||
state.closeEditMode = { [weak self] in
|
||
self?.lastInputEditCoordinator.close()
|
||
}
|
||
state.tapAIMic = { [weak self] in
|
||
self?.aiKeyboardCoordinator.toggleMicrophone()
|
||
}
|
||
state.cancelAIInput = { [weak self] in
|
||
self?.aiKeyboardCoordinator.cancel()
|
||
}
|
||
state.confirmPendingAIAnswer = { [weak self] in
|
||
self?.aiKeyboardCoordinator.confirmPendingAnswer()
|
||
}
|
||
state.selectAIReplyVariant = { [weak self] id in
|
||
self?.aiKeyboardCoordinator.selectReplyVariant(id: id)
|
||
}
|
||
state.discardPendingAIAnswer = { [weak self] in
|
||
self?.aiKeyboardCoordinator.discardPendingAnswer()
|
||
}
|
||
state.performAssistantFieldAction = { [weak self] in
|
||
self?.aiKeyboardCoordinator.performCurrentFieldAction()
|
||
}
|
||
state.submitAIHint = { [weak self] card in
|
||
self?.aiKeyboardCoordinator.submitHintCard(card)
|
||
}
|
||
state.submitAIClipboardSkill = { [weak self] skill, replyScene in
|
||
self?.aiKeyboardCoordinator.submitClipboardSkill(skill, replyScene: replyScene)
|
||
}
|
||
state.runClipboardExportSkill = { [weak self] skillID, titles in
|
||
AppGroupStore().setPendingShortcutRun(skillID: skillID, titles: titles)
|
||
AIAgentShortcutRun.trace("keyboard.openHost osgkeyboard://skill/run")
|
||
self?.openSkillShortcutRun()
|
||
}
|
||
state.openSettings = { [weak self] in self?.openHostApp() }
|
||
state.openInputMethodSetup = { [weak self] in self?.openHostApp(path: "deployrime") }
|
||
state.openClipboardSettings = { [weak self] in
|
||
SettingsDeepLink.setPending(.clipboard)
|
||
self?.openHostApp(path: "settings/clipboard")
|
||
}
|
||
state.tryAutoReplyFromGuide = { [weak self] in
|
||
self?.clipboardCapture.confirmAutoReplyGuide()
|
||
}
|
||
state.openClipboardPanel = { [weak self] in
|
||
self?.clipboardCapture.openPanelFromTopButton()
|
||
}
|
||
state.dismissClipboardOverlay = { [weak self] in
|
||
self?.clipboardCapture.dismissOverlay()
|
||
}
|
||
state.insertClipboardText = { [weak self] text in
|
||
guard let self else { return }
|
||
self.clipboardCapture.insertText(text) { insertText in
|
||
// Via the inserter so the undo key can roll a paste back.
|
||
self.textInserter.insertPasteboardText(insertText)
|
||
}
|
||
}
|
||
state.dismissClipboardSuggestion = { [weak self] in
|
||
self?.clipboardCapture.dismissSuggestion()
|
||
}
|
||
state.clearClipboardHistory = { [weak self] in
|
||
self?.clipboardCapture.clearHistory()
|
||
ClipboardReplyFeedbackStore.shared.clear()
|
||
}
|
||
state.deleteClipboardHistoryEntry = { [weak self] id in
|
||
self?.clipboardCapture.deleteEntry(id: id)
|
||
}
|
||
state.noteUserDidInputText = { [weak self] in
|
||
self?.clipboardCapture.noteUserDidInputText()
|
||
}
|
||
// The globe UIButton registers this controller's standard
|
||
// `handleInputModeList(from:with:)` action for all touch events.
|
||
state.inputModeController = self
|
||
state.startFlowSession = { [weak self] in self?.flowCoordinator.beginFlowStart() }
|
||
state.setMode = { [weak self] m in self?.configSync.persistMode(m) }
|
||
state.setLocale = { [weak self] l in self?.configSync.persistLocale(l) }
|
||
state.setEngineMode = { [weak self] m in self?.configSync.persistEngineMode(m) }
|
||
state.setTranslationTargetLocaleId = { [weak self] id in
|
||
self?.configSync.persistTranslationTargetLocaleId(id)
|
||
self?.aiKeyboardCoordinator.resetConversationForConfigurationChange()
|
||
}
|
||
state.insertNewline = { [weak self] in
|
||
self?.insertTextIntoDocument("\n", source: .manualKeyboard)
|
||
}
|
||
state.insertSpace = { [weak self] in
|
||
self?.insertTextIntoDocument(" ", source: .manualKeyboard)
|
||
}
|
||
state.deleteBackward = { [weak self] in self?.deleteBackwardFromDocument() }
|
||
state.undoLastInsertion = { [weak self] in self?.textInserter.undoLastInsertion() }
|
||
state.redoLastInsertion = { [weak self] in self?.textInserter.redoLastInsertion() }
|
||
state.copySelection = { [weak self] in self?.textInserter.copySelection() }
|
||
state.cutSelection = { [weak self] in self?.textInserter.cutSelection() }
|
||
state.setSurface = { [weak self] surface in
|
||
self?.applySurface(surface)
|
||
}
|
||
}
|
||
|
||
private func installSurfaceObservers() {
|
||
state.$phase
|
||
.receive(on: RunLoop.main)
|
||
.sink { [weak self] _ in
|
||
guard let self else { return }
|
||
if self.state.locksTypingSurface, self.state.surface == .typing {
|
||
self.applySurface(.voice)
|
||
}
|
||
}
|
||
.store(in: &cancellables)
|
||
|
||
state.$surface
|
||
.receive(on: RunLoop.main)
|
||
.sink { [weak self] surface in
|
||
guard let self else { return }
|
||
if surface == .typing {
|
||
self.activateTypingPresentationIfNeeded()
|
||
} else {
|
||
self.deactivateTypingSurface()
|
||
}
|
||
self.refreshKeyboardHeight()
|
||
}
|
||
.store(in: &cancellables)
|
||
}
|
||
|
||
/// This App Group property is supplied by the parallel settings module.
|
||
/// Reading it here keeps the shared state independent from persistence.
|
||
private func refreshMultipleReplyVariantsSetting() {
|
||
state.multipleReplyVariantsEnabled = AppGroupStore().multipleReplyVariantsEnabled
|
||
}
|
||
|
||
private func applySurface(_ requestedSurface: State.Surface) {
|
||
// `.ai` is retained only to decode preferences written by older builds.
|
||
// The product now has one unified assistant surface.
|
||
let surface: State.Surface = requestedSurface == .ai ? .voice : requestedSurface
|
||
if surface == .typing, state.locksTypingSurface {
|
||
OSGDiag.log("applySurface blocked typing (locksTypingSurface)", category: "boot")
|
||
return
|
||
}
|
||
if surface == .typing, FlowSessionBridge.isHostHeavy() {
|
||
OSGDiag.log("applySurface stay voice hostHeavy=1", category: "boot")
|
||
return
|
||
}
|
||
guard state.surface != surface else {
|
||
if surface == .typing {
|
||
activateTypingPresentationIfNeeded()
|
||
}
|
||
refreshKeyboardHeight()
|
||
return
|
||
}
|
||
OSGDiag.log(
|
||
"applySurface \(state.surface.rawValue) → \(surface.rawValue) \(OSGDiag.memoryTag())",
|
||
category: "boot"
|
||
)
|
||
state.surface = surface
|
||
recordMemory("KVC.applySurface", details: "requested=\(surface.rawValue)")
|
||
if surface == .typing {
|
||
activateTypingPresentationIfNeeded()
|
||
} else {
|
||
deactivateTypingSurface()
|
||
}
|
||
refreshKeyboardHeight()
|
||
}
|
||
|
||
private func activateTypingPresentationIfNeeded() {
|
||
guard isKeyboardPresentationActive,
|
||
state.surface == .typing,
|
||
typingDocumentPresentationID == nil else {
|
||
return
|
||
}
|
||
if !isTypingSurfaceActive {
|
||
typingSession.enterTypingMode()
|
||
isTypingSurfaceActive = true
|
||
}
|
||
typingDocumentPresentationID = typingSession.beginDocumentPresentation()
|
||
refreshEnglishSupplementaryLexicon()
|
||
if heightPhase == .presented {
|
||
refreshReturnKeyRole()
|
||
scheduleTypingDocumentSynchronization()
|
||
}
|
||
}
|
||
|
||
private func deactivateTypingSurface() {
|
||
guard isTypingSurfaceActive || typingDocumentPresentationID != nil else { return }
|
||
typingDocumentPresentationID = nil
|
||
typingSessionStorage?.leaveTypingMode()
|
||
isTypingSurfaceActive = false
|
||
}
|
||
|
||
private func scheduleTypingDocumentSynchronization() {
|
||
guard let presentationID = typingDocumentPresentationID else { return }
|
||
DispatchQueue.main.async { [weak self] in
|
||
guard let self,
|
||
self.isKeyboardPresentationActive,
|
||
self.state.surface == .typing,
|
||
self.typingDocumentPresentationID == presentationID else {
|
||
return
|
||
}
|
||
self.synchronizeTypingDocumentContext(
|
||
caretMoved: true,
|
||
presentationID: presentationID
|
||
)
|
||
}
|
||
}
|
||
|
||
private func synchronizeTypingDocumentContext(
|
||
caretMoved: Bool = false,
|
||
presentationID: UInt64? = nil
|
||
) {
|
||
guard let activeID = presentationID ?? typingDocumentPresentationID,
|
||
typingDocumentPresentationID == activeID else {
|
||
return
|
||
}
|
||
typingSessionStorage?.synchronizeEnglishDocumentContext(
|
||
caretMoved: caretMoved,
|
||
presentationID: activeID
|
||
)
|
||
}
|
||
|
||
private func applyPreferredSurfaceOnOpen() {
|
||
let preference = TypingInputConfiguration.preferredOpenPreference()
|
||
let resolved = KeyboardOpenSurfacePolicy.resolve(
|
||
locksTypingSurface: state.locksTypingSurface,
|
||
preferred: preference.surface
|
||
)
|
||
OSGDiag.log(
|
||
"applyPreferredSurfaceOnOpen preferred=\(preference.surface.rawValue) "
|
||
+ "resolved=\(resolved.rawValue) "
|
||
+ "lang=\(preference.typingLanguage?.rawValue ?? "-") "
|
||
+ "locksTyping=\(state.locksTypingSurface ? 1 : 0)",
|
||
category: "boot"
|
||
)
|
||
if resolved == .typing, let language = preference.typingLanguage {
|
||
_ = typingSession.setLanguage(language)
|
||
}
|
||
applySurface(resolved)
|
||
}
|
||
|
||
/// When not remembering, snap to the static open preference while hidden
|
||
/// so a reused keyboard instance does not animate voice → typing on show.
|
||
private func prepareSurfaceForNextPresentation() {
|
||
guard !TypingInputConfiguration.remembersLastSurface() else { return }
|
||
let preference = TypingInputConfiguration.preferredOpenPreference()
|
||
if state.surface != preference.surface {
|
||
state.surface = preference.surface
|
||
}
|
||
if preference.surface == .typing, let language = preference.typingLanguage {
|
||
_ = typingSession.setLanguage(language)
|
||
}
|
||
}
|
||
|
||
private func refreshKeyboardHeight() {
|
||
// Avoid synchronous layout here: this is also called during
|
||
// `viewDidLoad`, where re-entrant layout can observe partially
|
||
// initialized controller dependencies.
|
||
lockPresentedKeyboardHeight()
|
||
}
|
||
|
||
private func refreshLayoutMode() {
|
||
let usesIPadMetrics = isIPadLayout
|
||
let width = currentLayoutWidth
|
||
// `state.layoutWidth` tracks the width that last changed the layout
|
||
// bucket, not every intermediate width: republishing on each frame of
|
||
// a Stage Manager drag would rebuild the SwiftUI grid continuously.
|
||
let bucketChanged = KeyboardChromeLayout.usesWideIPadMetrics(
|
||
isIPad: usesIPadMetrics,
|
||
width: width
|
||
) != KeyboardChromeLayout.usesWideIPadMetrics(
|
||
isIPad: state.usesIPadLayoutMetrics,
|
||
width: state.layoutWidth
|
||
)
|
||
guard state.usesIPadLayoutMetrics != usesIPadMetrics || bucketChanged else { return }
|
||
state.usesIPadLayoutMetrics = usesIPadMetrics
|
||
state.layoutWidth = width
|
||
refreshKeyboardHeight()
|
||
}
|
||
|
||
private var heightPhaseLog: String {
|
||
switch heightPhase {
|
||
case .idle: return "idle"
|
||
case .presented: return "presented"
|
||
}
|
||
}
|
||
|
||
private func lockPresentedKeyboardHeight() {
|
||
keyboardHeightConstraint?.constant = targetKeyboardHeight
|
||
view.setNeedsLayout()
|
||
}
|
||
|
||
/// The constraint must stay at `target` once presented, whatever the system
|
||
/// did to the input view's height during the transition.
|
||
private func enforcePresentedKeyboardHeightIfNeeded() {
|
||
guard heightPhase == .presented else { return }
|
||
let target = targetKeyboardHeight
|
||
guard let constraint = keyboardHeightConstraint else { return }
|
||
let was = constraint.constant
|
||
guard abs(was - target) > 0.5 else { return }
|
||
constraint.constant = target
|
||
OSGDiag.log(
|
||
"KVC.heightEnforce constraint \(Int(was))→\(Int(target))",
|
||
category: "boot"
|
||
)
|
||
}
|
||
|
||
/// Keeps field-action UI current even when a host app does not immediately
|
||
/// echo this keyboard's own document mutation through `textDidChange`.
|
||
private func insertTextIntoDocument(
|
||
_ text: String,
|
||
source: KeyboardTextInsertionSource
|
||
) {
|
||
guard !text.isEmpty else { return }
|
||
lastTypingActivityUptime = ProcessInfo.processInfo.systemUptime
|
||
textDocumentProxy.insertText(text)
|
||
if source.contributesToKeyboardUsage {
|
||
let counts = KeyboardUsageCharacterClassifier.classify(text)
|
||
AnalyticsExtensionService.shared.keyboardUsageRecorder
|
||
.recordManualKeyboardCounts(
|
||
counts,
|
||
sessionID: keyboardUsageSessionID
|
||
)
|
||
}
|
||
// A non-empty insertion makes content actions available immediately.
|
||
// The next host callback remains the authoritative correction.
|
||
refreshAssistantFieldAction(hasTextOverride: true)
|
||
}
|
||
|
||
private func deleteBackwardFromDocument() {
|
||
lastTypingActivityUptime = ProcessInfo.processInfo.systemUptime
|
||
textDocumentProxy.deleteBackward()
|
||
refreshAssistantFieldAction()
|
||
scheduleAssistantFieldActionRefresh()
|
||
}
|
||
|
||
private func performDocumentReturn() {
|
||
insertTextIntoDocument("\n", source: .assistantAction)
|
||
scheduleAssistantFieldActionRefresh()
|
||
}
|
||
|
||
private func scheduleAssistantFieldActionRefresh() {
|
||
assistantFieldActionRefreshTask?.cancel()
|
||
assistantFieldActionRefreshTask = Task { @MainActor [weak self] in
|
||
try? await Task.sleep(for: .milliseconds(100))
|
||
guard !Task.isCancelled, let self else { return }
|
||
self.refreshAssistantFieldAction()
|
||
self.assistantFieldActionRefreshTask = nil
|
||
}
|
||
}
|
||
|
||
private func refreshAssistantFieldAction(hasTextOverride: Bool? = nil) {
|
||
let role = returnKeyRole(for: textDocumentProxy.returnKeyType ?? .default)
|
||
state.returnKeyRole = role
|
||
state.assistantActionAvailable = role.assistantActionAvailable(
|
||
hasText: hasTextOverride ?? textDocumentProxy.hasText
|
||
)
|
||
}
|
||
|
||
private func refreshReturnKeyRole() {
|
||
assistantFieldActionRefreshTask?.cancel()
|
||
assistantFieldActionRefreshTask = nil
|
||
refreshAssistantFieldAction()
|
||
// An unavailable trait is not proof that the new host field is safe.
|
||
// Fail closed until UIKit supplies an explicit non-secure value.
|
||
let isSecure = textDocumentProxy.isSecureTextEntry ?? true
|
||
state.setSecureTextEntry(isSecure)
|
||
clipboardCapture?.secureEntryDidChange(isSecure: isSecure)
|
||
// Secure fields must not run English autocomplete / autocorrect / learning.
|
||
typingSessionStorage?.suggestionsEnabled = !isSecure
|
||
if !isSecure {
|
||
typingSessionStorage?.syncAutocapitalization()
|
||
}
|
||
}
|
||
|
||
private func installTypingContextProviders() {
|
||
typingSession.precedingTextProvider = { [weak self] in
|
||
guard let self, self.textDocumentProxy.isSecureTextEntry == false else { return nil }
|
||
return self.textDocumentProxy.documentContextBeforeInput
|
||
}
|
||
typingSession.followingTextProvider = { [weak self] in
|
||
guard let self, self.textDocumentProxy.isSecureTextEntry == false else { return nil }
|
||
return self.textDocumentProxy.documentContextAfterInput
|
||
}
|
||
typingSession.autocapitalizationModeProvider = { [weak self] in
|
||
guard let self, self.textDocumentProxy.isSecureTextEntry == false else { return .none }
|
||
return Self.typingAutocapitalizationMode(
|
||
for: self.textDocumentProxy.autocapitalizationType ?? .sentences
|
||
)
|
||
}
|
||
}
|
||
|
||
/// Contacts and user text replacements, without a Contacts permission.
|
||
///
|
||
/// `requestSupplementaryLexicon` replies on `com.apple.TextInput.lexicon-request`,
|
||
/// not the main actor. Touching `TypingSessionController` there traps in Swift 6
|
||
/// (`_dispatch_assert_queue_fail`) and the extension is killed on appear.
|
||
private func refreshEnglishSupplementaryLexicon() {
|
||
requestSupplementaryLexicon { @Sendable lexicon in
|
||
Task { @MainActor [weak self] in
|
||
self?.applySupplementaryLexicon(lexicon)
|
||
}
|
||
}
|
||
}
|
||
|
||
private func applySupplementaryLexicon(_ lexicon: UILexicon) {
|
||
typingSessionStorage?.supplementaryWords = lexicon.entries.compactMap { entry -> String? in
|
||
let text = entry.documentText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||
guard !text.isEmpty else { return nil }
|
||
let parts = text.split { $0.isWhitespace || $0 == "," }
|
||
guard parts.count == 1 else { return nil }
|
||
let token = String(parts[0])
|
||
guard token.allSatisfy({ $0.isLetter || $0 == "'" || $0 == "’" || $0 == "-" }) else {
|
||
return nil
|
||
}
|
||
return token
|
||
}
|
||
}
|
||
|
||
private static func typingAutocapitalizationMode(
|
||
for type: UITextAutocapitalizationType
|
||
) -> TypingAutocapitalizationMode {
|
||
switch type {
|
||
case .none:
|
||
return .none
|
||
case .words:
|
||
return .words
|
||
case .sentences:
|
||
return .sentences
|
||
case .allCharacters:
|
||
return .allCharacters
|
||
@unknown default:
|
||
return .sentences
|
||
}
|
||
}
|
||
|
||
private func returnKeyRole(for returnKeyType: UIReturnKeyType) -> State.ReturnKeyRole {
|
||
switch returnKeyType {
|
||
case .default:
|
||
return .newline
|
||
case .go:
|
||
return .go
|
||
case .google:
|
||
return .google
|
||
case .join:
|
||
return .join
|
||
case .next:
|
||
return .next
|
||
case .route:
|
||
return .route
|
||
case .search:
|
||
return .search
|
||
case .send:
|
||
return .send
|
||
case .yahoo:
|
||
return .yahoo
|
||
case .done:
|
||
return .done
|
||
case .emergencyCall:
|
||
return .emergencyCall
|
||
case .continue:
|
||
return .continue
|
||
@unknown default:
|
||
return .newline
|
||
}
|
||
}
|
||
|
||
// MARK: - System keyboard chrome
|
||
|
||
private func configureDictationBehavior() {
|
||
hasDictationKey = true
|
||
}
|
||
|
||
/// Only walk our own input-view subtree. Recursing into the host window /
|
||
/// root VC previously risked "System gesture gate timed out" and the
|
||
/// system killing the keyboard plugin.
|
||
private func disableSystemGestureDelays() {
|
||
disableGestureDelays(in: view)
|
||
}
|
||
|
||
private func disableGestureDelays(in targetView: UIView) {
|
||
targetView.gestureRecognizers?.forEach { recognizer in
|
||
recognizer.delaysTouchesBegan = false
|
||
recognizer.delaysTouchesEnded = false
|
||
recognizer.cancelsTouchesInView = false
|
||
if recognizer is UIScreenEdgePanGestureRecognizer {
|
||
recognizer.isEnabled = false
|
||
}
|
||
}
|
||
targetView.subviews.forEach(disableGestureDelays)
|
||
}
|
||
|
||
// MARK: - Layout
|
||
|
||
private func installKeyboardHeight() {
|
||
let constraint = view.heightAnchor.constraint(equalToConstant: targetKeyboardHeight)
|
||
constraint.priority = UILayoutPriority(999)
|
||
constraint.isActive = true
|
||
keyboardHeightConstraint = constraint
|
||
}
|
||
|
||
/// Every height constraint the system and we put on the input view. The
|
||
/// system's own constant walks from the full screen height down to the
|
||
/// keyboard slot during the slide-in, so this is what to check whenever the
|
||
/// surface appears mis-sized or off-slot.
|
||
private func logHeightConstraints(tag: String) {
|
||
let heights = view.constraints.filter { constraint in
|
||
constraint.firstItem as? UIView === view && constraint.firstAttribute == .height
|
||
}
|
||
let described = heights.map { constraint in
|
||
let name = constraint === keyboardHeightConstraint
|
||
? "ours"
|
||
: (constraint.identifier ?? "system")
|
||
return "\(name)=\(Int(constraint.constant))@\(Int(constraint.priority.rawValue))"
|
||
+ (constraint.isActive ? "" : "(inactive)")
|
||
}
|
||
OSGDiag.log(
|
||
"KVC.heightConstraints[\(tag)] \(described.joined(separator: " "))",
|
||
category: "boot"
|
||
)
|
||
}
|
||
|
||
private func logLayoutSnapshotIfChanged() {
|
||
let snapshot = "phase=\(heightPhaseLog) "
|
||
+ "view=\(Int(view.bounds.height)) "
|
||
+ "host=\(Int(hosting?.view.bounds.height ?? -1)) "
|
||
+ "constraint=\(Int(keyboardHeightConstraint?.constant ?? -1)) "
|
||
+ "target=\(Int(targetKeyboardHeight))"
|
||
guard snapshot != lastLoggedLayoutSnapshot else { return }
|
||
lastLoggedLayoutSnapshot = snapshot
|
||
OSGDiag.log("KVC.layout \(snapshot)", category: "boot")
|
||
}
|
||
|
||
private func installSwiftUI() {
|
||
let root = KeyboardSurfaceRoot(
|
||
state: state,
|
||
typing: typingSession,
|
||
onInsert: { [weak self] text in
|
||
self?.insertTextIntoDocument(text, source: .manualKeyboard)
|
||
},
|
||
onDeleteBackward: { [weak self] in
|
||
self?.deleteBackwardFromDocument()
|
||
}
|
||
)
|
||
let host = KeyboardHostingController(rootView: root)
|
||
host.view.backgroundColor = .clear
|
||
host.view.translatesAutoresizingMaskIntoConstraints = false
|
||
host.view.clipsToBounds = false
|
||
host.view.insetsLayoutMarginsFromSafeArea = false
|
||
host.safeAreaRegions = []
|
||
addChild(host)
|
||
view.addSubview(host.view)
|
||
NSLayoutConstraint.activate([
|
||
host.view.leadingAnchor.constraint(equalTo: view.leadingAnchor),
|
||
host.view.trailingAnchor.constraint(equalTo: view.trailingAnchor),
|
||
host.view.topAnchor.constraint(equalTo: view.topAnchor),
|
||
host.view.bottomAnchor.constraint(equalTo: view.bottomAnchor)
|
||
])
|
||
host.didMove(toParent: self)
|
||
hosting = host
|
||
}
|
||
|
||
// MARK: - App context
|
||
|
||
private func detectAndStoreAppContext() {
|
||
guard textDocumentProxy.isSecureTextEntry == false else { return }
|
||
let preceding = textDocumentProxy.documentContextBeforeInput
|
||
let store = AppGroupStore()
|
||
let detector = AppContextDetector()
|
||
let context = detector.detect(
|
||
precedingText: preceding,
|
||
storedCache: store.detectedAppContext
|
||
)
|
||
store.setDetectedAppContext(context)
|
||
}
|
||
|
||
private func captureFieldContext() -> FlowFieldContext {
|
||
let isSecure = textDocumentProxy.isSecureTextEntry ?? true
|
||
let preceding = isSecure ? nil : textDocumentProxy.documentContextBeforeInput
|
||
let following = isSecure ? nil : textDocumentProxy.documentContextAfterInput
|
||
let isAvailable = preceding != nil || following != nil
|
||
let isEmpty = isAvailable && (preceding ?? "").isEmpty && (following ?? "").isEmpty
|
||
|
||
return FlowFieldContext(
|
||
precedingText: preceding.map { String($0.suffix(600)) },
|
||
followingText: following.map { String($0.prefix(200)) },
|
||
keyboardType: keyboardTypeName(textDocumentProxy.keyboardType ?? .default),
|
||
returnKeyType: returnKeyTypeName(textDocumentProxy.returnKeyType ?? .default),
|
||
isSecureEntry: isSecure,
|
||
isEmptyField: isEmpty,
|
||
isContextAvailable: isAvailable
|
||
)
|
||
}
|
||
|
||
private func keyboardTypeName(_ type: UIKeyboardType) -> String {
|
||
switch type {
|
||
case .asciiCapable: return "asciiCapable"
|
||
case .numbersAndPunctuation: return "numbersAndPunctuation"
|
||
case .URL: return "url"
|
||
case .numberPad: return "numberPad"
|
||
case .phonePad: return "phonePad"
|
||
case .namePhonePad: return "namePhonePad"
|
||
case .emailAddress: return "emailAddress"
|
||
case .decimalPad: return "decimalPad"
|
||
case .twitter: return "twitter"
|
||
case .webSearch: return "webSearch"
|
||
case .asciiCapableNumberPad: return "asciiCapableNumberPad"
|
||
case .default: return "default"
|
||
@unknown default: return "default"
|
||
}
|
||
}
|
||
|
||
private func returnKeyTypeName(_ type: UIReturnKeyType) -> String {
|
||
switch type {
|
||
case .go: return "go"
|
||
case .google: return "google"
|
||
case .join: return "join"
|
||
case .next: return "next"
|
||
case .route: return "route"
|
||
case .search: return "search"
|
||
case .send: return "send"
|
||
case .yahoo: return "yahoo"
|
||
case .done: return "done"
|
||
case .emergencyCall: return "emergencyCall"
|
||
case .continue: return "continue"
|
||
case .default: return "default"
|
||
@unknown default: return "default"
|
||
}
|
||
}
|
||
|
||
// MARK: - Open host app
|
||
|
||
private func openWebURL(_ url: URL) {
|
||
guard hasFullAccess else {
|
||
state.skillTipText = ExtL10n.string("keyboard.error.fullAccessForJump")
|
||
return
|
||
}
|
||
HostAppLauncher.open(url: url, from: self) { [weak self] success in
|
||
guard !success else { return }
|
||
self?.state.skillTipText = ExtL10n.string("keyboard.ai.skill.openLinkFailed")
|
||
}
|
||
}
|
||
|
||
private func callPhone(_ phoneNumber: String) {
|
||
guard hasFullAccess else {
|
||
state.skillTipText = ExtL10n.string("keyboard.error.fullAccessForJump")
|
||
return
|
||
}
|
||
guard let url = AIPhoneNumberResolver.telephoneURL(for: phoneNumber) else {
|
||
state.skillTipText = ExtL10n.string("keyboard.ai.skill.callPhoneFailed")
|
||
return
|
||
}
|
||
HostAppLauncher.open(url: url, from: self) { [weak self] success in
|
||
guard !success else { return }
|
||
self?.state.skillTipText = ExtL10n.string("keyboard.ai.skill.callPhoneFailed")
|
||
}
|
||
}
|
||
|
||
private func createContact(_ phoneNumber: String) {
|
||
guard hasFullAccess else {
|
||
state.skillTipText = ExtL10n.string("keyboard.error.fullAccessForJump")
|
||
return
|
||
}
|
||
AppGroupStore().setPendingContactCreation(phoneNumber: phoneNumber)
|
||
guard let url = URL(string: "osgkeyboard://contact/new") else { return }
|
||
HostAppLauncher.open(url: url, from: self) { [weak self] success in
|
||
guard !success else { return }
|
||
self?.state.skillTipText = ExtL10n.string(
|
||
"keyboard.ai.skill.createContactFailed"
|
||
)
|
||
}
|
||
}
|
||
|
||
private func openSkillShortcutRun() {
|
||
guard hasFullAccess else {
|
||
state.skillTipText = ExtL10n.string("keyboard.error.fullAccessForJump")
|
||
return
|
||
}
|
||
guard let url = URL(string: "osgkeyboard://skill/run") else { return }
|
||
HostAppLauncher.open(url: url, from: self) { [weak self] success in
|
||
AIAgentShortcutRun.trace("keyboard.openHost result success=\(success)")
|
||
if success { return }
|
||
self?.state.skillTipText = ExtL10n.string("keyboard.ai.skill.handoffFailed")
|
||
}
|
||
}
|
||
|
||
private func openHostApp(path: String = "settings") {
|
||
guard hasFullAccess else {
|
||
let msg = ExtL10n.string("keyboard.error.fullAccessForJump")
|
||
state.phase = .error(.manualOpenRequired, message: msg)
|
||
scheduleAutoClearError()
|
||
return
|
||
}
|
||
guard let url = URL(string: "osgkeyboard://\(path)") else {
|
||
flowCoordinator.handleHostAppOpenResult(path: path, success: false)
|
||
return
|
||
}
|
||
HostAppLauncher.open(url: url, from: self) { [weak self] success in
|
||
self?.flowCoordinator.handleHostAppOpenResult(path: path, success: success)
|
||
}
|
||
}
|
||
|
||
private func scheduleAutoClearError() {
|
||
Task { @MainActor [weak self] in
|
||
try? await Task.sleep(nanoseconds: 2_400_000_000)
|
||
guard let self else { return }
|
||
switch self.state.phase {
|
||
case .error:
|
||
self.state.phase = .idle
|
||
// Re-derive mic availability right away so a now-ready host
|
||
// turns the mic green immediately instead of lingering orange
|
||
// until the next monitor tick.
|
||
self.flowCoordinator.refreshSessionState()
|
||
default:
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|