rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: harden Flow cold-start/force-quit and polish macOS dictation UX

Browse Source 
Fix cold-start overlay recursion that overflowed the main-thread stack when
recording began while the ready overlay was still up; also remove temporary
on-screen Flow DEBUG panels after the orange-mic investigation, and land the
macOS overlay/catalog/layout polish plus related Flow recovery hardening.
This commit is contained in:
Rocky
2026-07-10 12:39:41 +08:00
parent dcb66a9849
commit cdf833935a
104 changed files with 5794 additions and 853 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OSGKeyboardExt/KeyboardViewController.swift
+7 -1
View File
@@ -57,7 +57,11 @@ public final class KeyboardViewController: UIInputViewController {
// Voice-first keyboard — hide the misleading "English" subtitle in Settings.
primaryLanguage = "mis"
OSGLog.keyboardExt.info("viewDidLoad — extension booted")
CustomLanguageModelManager.shared.prepareInBackgroundIfNeeded()
// Deliberately NO CustomLanguageModelManager prewarm here: the
// extension never runs ASR (the host app owns the microphone and
// the SpeechAnalyzer pipeline), and compiling/caching an LM inside
// the keyboard's ~60 MB jetsam budget risks the system killing the
// keyboard outright. The host app prewarms it on session start.
setNeedsUpdateOfScreenEdgesDeferringSystemGestures()
installKeyboardHeight()
configureDictationBehavior()
@@ -84,6 +88,7 @@ public final class KeyboardViewController: UIInputViewController {
setNeedsUpdateOfScreenEdgesDeferringSystemGestures()
configureDictationBehavior()
KeyboardSetupBridge.markExtensionAppearance(hasFullAccess: hasFullAccess)
state.debugHasFullAccess = hasFullAccess
flowCoordinator.refreshSessionState()
flowCoordinator.startSessionMonitor()
configSync.syncOnboardingStateFromAppGroup()
@@ -126,6 +131,7 @@ public final class KeyboardViewController: UIInputViewController {
textInserter = KeyboardTextInserter(
state: state,
insertText: { [weak self] text in self?.textDocumentProxy.insertText(text) },
contextBeforeInput: { [weak self] in self?.textDocumentProxy.documentContextBeforeInput },
scheduleAutoClearError: { [weak self] in self?.scheduleAutoClearError() }
)
OSGKeyboardExt/Services/HostAppLauncher.swift
+51 -10
View File
@@ -4,15 +4,17 @@
// Opens the host app from the keyboard extension.
//
// Reality check (verified against iOS 18–26 behaviour):
// • `extensionContext.open` is documented for Today widgets only; for a
// keyboard extension it resolves `false`, so we do not use it.
// • The deprecated `openURL:` selector hack was disabled in iOS 18
// ("BUG IN CLIENT OF UIKIT … migrate to open(_:options:completionHandler:)").
// • The still-working path is: walk the responder chain to `UIApplication`
// and call the non-deprecated `open(_:options:completionHandler:)`. This
// requires Full Access and grows less reliable on newer iOS, so we report
// the *real* success from the completion handler instead of assuming it
// worked — callers degrade to on-keyboard guidance when it returns false.
// • Primary path: walk the responder chain to `UIApplication` and call the
// non-deprecated `open(_:options:completionHandler:)`. This requires Full
// Access and grows less reliable on newer iOS, so we report the *real*
// success from the completion handler instead of assuming it worked.
// • Fallback: `extensionContext.open`. Historically documented for Today
// widgets only (and it used to resolve `false` for keyboards), but it is
// the Apple-documented API for extensions to open URLs and ships in
// production keyboards on current iOS — worth trying before giving up.
// When both paths fail, callers degrade to on-keyboard guidance.
import UIKit
@@ -27,13 +29,52 @@ enum HostAppLauncher {
while let current = responder {
if let application = current as? UIApplication {
application.open(url, options: [:]) { success in
Task { @MainActor in completion(success) }
Task { @MainActor in
if success {
completion(true)
} else {
openViaExtensionContext(url: url, from: controller, completion: completion)
}
}
}
return
}
responder = current.next
}
// No `UIApplication` in the responder chain — cannot open the host app.
completion(false)
// No `UIApplication` in the responder chain — try the extension context.
openViaExtensionContext(url: url, from: controller, completion: completion)
}
@MainActor
private static func openViaExtensionContext(
url: URL,
from controller: KeyboardViewController,
completion: @escaping @MainActor (Bool) -> Void
) {
guard let context = controller.extensionContext else {
completion(false)
return
}
// `NSExtensionContext.open` from keyboards has historically been
// flaky about ever invoking its completion on some iOS versions.
// Callers rely on a real answer to fail fast (instead of spinning
// the 30 s start watchdog), so race the callback against a timeout
// and report the first result only.
var didComplete = false
let finish: @MainActor (Bool) -> Void = { success in
guard !didComplete else { return }
didComplete = true
completion(success)
}
Task { @MainActor in
// 1.5 s: long enough for a real open to call back, short enough
// that a dead completion degrades to on-keyboard guidance before
// the user gives up staring at nothing.
try? await Task.sleep(nanoseconds: 1_500_000_000)
finish(false)
}
context.open(url) { success in
Task { @MainActor in finish(success) }
}
}
}
OSGKeyboardExt/Services/KeyboardFlowCoordinator.swift
+111 -1
View File
@@ -44,6 +44,10 @@ final class KeyboardFlowCoordinator {
private var isAwaitingFlowResult = false
private var activeSessionId: UUID?
private var currentUtteranceId: UUID?
/// Utterance whose final result we already inserted (or failed). Prevents
/// `adoptHostBusyStateIfNeeded` from re-entering `.processing` after a
/// stale App Group snapshot still says `reason=processing`.
private var lastConsumedUtteranceId: UUID?
private var currentCommandSeq: Int64 = 0
private var lastAvailabilityTraceSignature = ""
@@ -139,6 +143,12 @@ final class KeyboardFlowCoordinator {
FlowSessionBridge.reloadFromDisk()
let readySnapshot = FlowSessionBridge.readySnapshot()
activeSessionId = readySnapshot?.sessionId ?? activeSessionId
// If the host is mid-utterance but this extension process lost local
// ownership (jetsam / recreate after app switch), re-adopt it so we
// show red/white instead of a fake orange "starting" state.
adoptHostBusyStateIfNeeded(snapshot: readySnapshot)
let hostReady = readySnapshot?.ready == true && FlowSessionBridge.isHostReady()
let now = Date().timeIntervalSince1970
if hostReady { lastHostReadyAt = now }
@@ -148,10 +158,20 @@ final class KeyboardFlowCoordinator {
// across cross-process read jitter and anchors this smoothing.
let withinReadyGrace = lastHostReadyAt > 0
&& (now - lastHostReadyAt) <= Self.hostReadyGrace
// Host busy (recording/processing) is NOT "still starting". Treating
// it as preparingSession was the orange-stuck bug after cold start:
// host utt.rec=1 → ready=false → keyboard forever "正在启动…".
let hostBusy = readySnapshot?.reason == .recording
|| readySnapshot?.reason == .processing
let hostWarming = !hostReady
&& !hostBusy
&& FlowSessionBridge.isSessionActive()
&& (FlowSessionBridge.isHostReachable() || isPendingFlowStart || withinReadyGrace)
state.flowSessionActive = FlowSessionBridge.isSessionActive()
state.debugPendingFlowStart = isPendingFlowStart
state.debugFlowRecording = isFlowRecording
state.debugAwaitingFlowResult = isAwaitingFlowResult
state.debugHasFullAccess = hasFullAccess()
state.micVoiceAvailability = MicVoiceAvailabilityResolver.resolve(
phase: state.phase,
micDisabled: state.micDisabled,
@@ -176,6 +196,84 @@ final class KeyboardFlowCoordinator {
}
}
/// Re-attach to a host utterance this keyboard process no longer owns.
private func adoptHostBusyStateIfNeeded(snapshot: FlowReadySnapshot?) {
guard let snapshot, let sessionId = snapshot.sessionId else { return }
// Ignore snapshots from a dead host generation.
if let snapGen = snapshot.hostGeneration,
let liveGen = FlowSessionBridge.currentHostGeneration(),
snapGen != liveGen {
return
}
// Host already finished — never re-adopt a consumed utterance, and
// clear sticky local processing left behind by a stale busy snapshot.
if snapshot.reason != .recording, snapshot.reason != .processing {
clearStickyProcessingIfNeeded(hostReady: snapshot.ready)
return
}
switch snapshot.reason {
case .recording:
guard !isFlowRecording else { return }
guard !isAwaitingFlowResult else { return }
// Require the host's utterance id — inventing one makes matchingResult
// forever miss the real delivery and leaves the mic white forever.
guard let busyId = snapshot.busyUtteranceId else { return }
guard busyId != lastConsumedUtteranceId else { return }
activeSessionId = sessionId
currentUtteranceId = busyId
isPendingFlowStart = false
flowStartDeadline = 0
stopHostReadyWait()
isFlowRecording = true
state.phase = .recording
if state.lastTranscript.isEmpty {
state.lastTranscript = ""
}
if let view = wakeLockView() {
ExtensionScreenWakeLock.acquire(from: view)
}
startUtteranceCountdown()
startFlowLevelWatchdog()
traceState("adoptHostBusy.recording", extra: "session=\(sessionId)")
case .processing:
guard !isAwaitingFlowResult else { return }
guard let busyId = snapshot.busyUtteranceId else { return }
guard busyId != lastConsumedUtteranceId else { return }
activeSessionId = sessionId
currentUtteranceId = busyId
isPendingFlowStart = false
flowStartDeadline = 0
isFlowRecording = false
stopUtteranceCountdown()
ExtensionScreenWakeLock.release()
state.phase = .processing
if state.lastTranscript.isEmpty {
state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
}
startFlowResultWatchdog()
traceState("adoptHostBusy.processing", extra: "session=\(sessionId)")
default:
break
}
}
/// After insert, a stale `reason=processing` snapshot can bounce the mic
/// back to white loading. When the host is no longer busy, force idle.
private func clearStickyProcessingIfNeeded(hostReady: Bool) {
guard !isAwaitingFlowResult, !isFlowRecording else { return }
guard case .processing = state.phase else { return }
state.phase = .idle
state.lastTranscript = ""
stopFlowWatchdog()
currentUtteranceId = nil
traceState(
"stickyProcessing.cleared",
extra: hostReady ? "hostReady=1" : "hostReady=0"
)
}
/// Session is live but the ready contract has not landed yet — poll
/// quickly instead of sticking on "session inactive" orange.
private func startHostReadyWaitIfNeeded() {
@@ -184,6 +282,12 @@ final class KeyboardFlowCoordinator {
stopHostReadyWait()
return
}
// Host busy ≠ waiting for ready. Do not spin the ready-wait poll.
if let reason = FlowSessionBridge.readySnapshot()?.reason,
reason == .recording || reason == .processing {
stopHostReadyWait()
return
}
guard !FlowSessionBridge.isHostReady() else {
stopHostReadyWait()
return
@@ -196,7 +300,9 @@ final class KeyboardFlowCoordinator {
guard let self, !Task.isCancelled else { return }
FlowSessionBridge.reloadFromDisk()
self.recomputeMicVoiceAvailability()
if self.state.micVoiceAvailability.isReady {
if self.state.micVoiceAvailability.isReady
|| self.state.micVoiceAvailability == .recording
|| self.state.micVoiceAvailability == .processing {
return
}
try? await Task.sleep(nanoseconds: 150_000_000)
@@ -365,6 +471,7 @@ final class KeyboardFlowCoordinator {
)
)
FlowSessionBridge.clearResult()
lastConsumedUtteranceId = result.utteranceId
currentUtteranceId = nil
textInserter.handleFlowTranscript(
TranscriptionDelivery(text: text, polishWarning: result.warning)
@@ -375,6 +482,7 @@ final class KeyboardFlowCoordinator {
isAwaitingFlowResult = false
stopFlowWatchdog()
FlowSessionBridge.clearResult()
lastConsumedUtteranceId = result.utteranceId
currentUtteranceId = nil
let error = FlowTranscriptionError(
message: result.text ?? ExtL10n.string("keyboard.flow.resultTimeout"),
@@ -626,6 +734,7 @@ final class KeyboardFlowCoordinator {
)
)
FlowSessionBridge.clearResult()
self.lastConsumedUtteranceId = result.utteranceId
self.currentUtteranceId = nil
self.debug("resultWatchdog consumed delivery len=\(text.count)")
self.textInserter.handleFlowTranscript(
@@ -637,6 +746,7 @@ final class KeyboardFlowCoordinator {
self.isAwaitingFlowResult = false
self.stopFlowWatchdog()
FlowSessionBridge.clearResult()
self.lastConsumedUtteranceId = result.utteranceId
self.currentUtteranceId = nil
let error = FlowTranscriptionError(
message: result.text ?? ExtL10n.string("keyboard.flow.resultTimeout"),
OSGKeyboardExt/Services/KeyboardTextInserter.swift
+10 -1
View File
@@ -10,15 +10,18 @@ import OSGKeyboardShared
final class KeyboardTextInserter {
private let state: KeyboardState
private let insertText: (String) -> Void
private let contextBeforeInput: () -> String?
private let scheduleAutoClearError: () -> Void
init(
state: KeyboardState,
insertText: @escaping (String) -> Void,
contextBeforeInput: @escaping () -> String?,
scheduleAutoClearError: @escaping () -> Void
) {
self.state = state
self.insertText = insertText
self.contextBeforeInput = contextBeforeInput
self.scheduleAutoClearError = scheduleAutoClearError
}
@@ -30,7 +33,13 @@ final class KeyboardTextInserter {
return
}
// Host app already polished when configured; keyboard only inserts.
insertText(trimmed)
// Word-boundary hygiene: dictating "world" with the cursor right
// after "Hello" must yield "Hello world", not "Helloworld".
let separator = DictationTextComposer.insertionSeparator(
previousContext: contextBeforeInput(),
insertion: trimmed
)
insertText(separator + trimmed)
state.lastTranscript = ""
state.level = 0
if let warning = delivery.polishWarning {
OSGKeyboardExt/Views/KeyboardRootView.swift
+7
View File
@@ -33,6 +33,11 @@ private enum KeyboardLayoutMetrics {
static let topBarToTranscriptSpacing: CGFloat = Spacing.xs / 2
/// Outer inset for the bottom action row from screen edges (8 pt → 24 pt, +200%).
static let sideActionHorizontalInset: CGFloat = Spacing.xs * 3
/// iPad: cap the content column. A full-width (~1180 pt) keyboard would
/// park delete/return at the far screen edges and turn each cursor-drag
/// pad into a ~450 pt runway — capping keeps the reach ergonomics of the
/// phone layout. iPhone widths are all below this, so it is a no-op there.
static let contentMaxWidth: CGFloat = 700
// MARK: - Content-driven keyboard height (single source of truth)
static let outerPaddingTop: CGFloat = 2
@@ -109,6 +114,8 @@ public struct KeyboardRootView: View {
.padding(.bottom, KeyboardLayoutMetrics.outerPaddingBottom)
// 透明背景：让系统键盘 chrome 透出，不自行铺色（深浅模式一致）。
.background(Color.clear)
.frame(maxWidth: KeyboardLayoutMetrics.contentMaxWidth)
.frame(maxWidth: .infinity)
.frame(height: Self.totalHeight)
// Feed the resolved palette to all nested chips/buttons.
.environment(\.themePalette, palette)