fix(ipad): ship iPad P0 layout/globe fixes, edit-last-input, drop clipboard commands

Adapt typing/voice surfaces for iPad width and height, add the system globe
key and last-input editing flow, harden host-only Rime deployment, and remove
clipboard voice commands. Bump build to 61.
This commit is contained in:
Rocky
2026-08-10 13:50:50 +08:00
parent 53abad2050
commit 2bc8c1b87d
85 changed files with 5703 additions and 3997 deletions
+21
View File
@@ -28,6 +28,26 @@ struct OSGKeyboardApp: App {
var body: some Scene {
WindowGroup {
#if DEBUG
if ProcessInfo.processInfo.arguments.contains("--edit-demo") {
EditDemoView()
} else if ProcessInfo.processInfo.arguments.contains("--edit-pager-ui-test") {
ThemedRoot {
EditPagerUITestHarness()
}
.preferredColorScheme(appearance.colorScheme)
} else if AppGroup.isAvailable {
ThemedRoot {
MainAppRoot()
}
.preferredColorScheme(appearance.colorScheme)
} else {
ThemedRoot {
AppGroupErrorView()
}
.preferredColorScheme(appearance.colorScheme)
}
#else
if AppGroup.isAvailable {
ThemedRoot {
MainAppRoot()
@@ -39,6 +59,7 @@ struct OSGKeyboardApp: App {
}
.preferredColorScheme(appearance.colorScheme)
}
#endif
}
}
}
+517 -93
View File
@@ -64,21 +64,31 @@ final class FlowSessionManager: ObservableObject {
private var heartbeatTask: Task<Void, Never>?
private var levelTask: Task<Void, Never>?
private var startTask: Task<Void, Never>?
private var audioPrimeTask: Task<Bool, Never>?
private var audioPrimeID: UUID?
private var audioPrimeCancellationRequested = false
private var startupAudioHealthTask: Task<Void, Never>?
private var didRunStartupAudioHealthCheck = false
private var commandObserver: FlowSessionDarwinObserver?
/// Last recording state the poll loop observed logs only on transition.
private var lastObservedRecordingState: FlowSessionKeys.RecordingState = .idle
private var activeSessionId: UUID?
private var currentUtteranceId: UUID?
/// Claimed synchronously before any async capture work begins.
private var startingUtteranceId: UUID?
private var startTransactionDeadlineAt: TimeInterval?
/// Monotonic token captured by every async worker for one utterance.
private var utteranceGeneration: UInt64 = 0
/// Terminal delivery is idempotent: only the first path may write a result.
private var terminalUtteranceIds: Set<UUID> = []
/// Cursor context captured by the keyboard at the final insertion point.
private var pendingFieldContext: FlowFieldContext?
/// Dictation vs clipboard-command for the live utterance (set on start).
/// Dictation vs explicit last-input editing for the live utterance.
private var currentUtteranceMode: FlowUtteranceMode = .dictation
private var pendingClipboardSnapshot: String?
private var pendingPreviousOutput: String?
private var pendingEditSourceText: String?
private var pendingSourceHistoryEntryID: UUID?
private var pendingSourceHistoryEntryRevision: Int64?
private var pendingProcessingDeadlineAt: TimeInterval?
private var pendingStopUtteranceId: UUID?
private var currentCommandSeq: Int64 = 0
private var lastHandledCommandSeq: Int64 = 0
@@ -110,11 +120,11 @@ final class FlowSessionManager: ObservableObject {
/// True while handling a keyboard-initiated `startflow` cold start.
private var isColdStartHandoff = false
private var coldStartRecoveryTask: Task<Void, Never>?
/// Initial proof window cold mic sessions often need >2.5s after app switch.
private static let coldStartAudioProofTimeout: TimeInterval = 6
var shouldDeferHostHeavyWork: Bool {
isUtteranceRecording || isUtteranceProcessing || hasUnacknowledgedTerminalResult()
startingUtteranceId != nil
|| isUtteranceRecording
|| isUtteranceProcessing
|| hasUnacknowledgedTerminalResult()
}
func attachPiPHostView(_ view: UIView) {
@@ -342,6 +352,12 @@ final class FlowSessionManager: ObservableObject {
coldStartRecoveryTask = nil
startTask?.cancel()
startTask = nil
startupAudioHealthTask?.cancel()
startupAudioHealthTask = nil
audioPrimeTask?.cancel()
audioPrimeTask = nil
audioPrimeID = nil
audioPrimeCancellationRequested = false
commandObserver = nil
pollingTask?.cancel()
pollingTask = nil
@@ -420,6 +436,12 @@ final class FlowSessionManager: ObservableObject {
coldStartRecoveryTask = nil
startTask?.cancel()
startTask = nil
startupAudioHealthTask?.cancel()
startupAudioHealthTask = nil
audioPrimeTask?.cancel()
audioPrimeTask = nil
audioPrimeID = nil
audioPrimeCancellationRequested = false
commandObserver = nil
pollingTask?.cancel()
pollingTask = nil
@@ -611,6 +633,7 @@ final class FlowSessionManager: ObservableObject {
let hasPendingDelivery = hasUnacknowledgedTerminalResult()
let canAcceptUtterance = pipController.isPictureInPictureActive
&& pollingAlive
&& startingUtteranceId == nil
&& !isUtteranceRecording
&& !isUtteranceProcessing
&& sessionWarning == nil
@@ -620,6 +643,8 @@ final class FlowSessionManager: ObservableObject {
let reason: FlowReadySnapshot.Reason
if canAcceptUtterance {
reason = .ready
} else if startingUtteranceId != nil {
reason = .waitingForAudioProof
} else if sessionWarning != nil {
reason = .error
} else if isUtteranceRecording {
@@ -650,9 +675,10 @@ final class FlowSessionManager: ObservableObject {
audioProofAt: hasRecentAudio ? now : nil,
engineMode: store.engineMode,
localeId: store.localeId,
busyUtteranceId: isUtteranceRecording || isUtteranceProcessing
? currentUtteranceId
: (hasPendingDelivery ? FlowSessionBridge.latestResult()?.utteranceId : nil),
busyUtteranceId: startingUtteranceId
?? (isUtteranceRecording || isUtteranceProcessing
? currentUtteranceId
: (hasPendingDelivery ? FlowSessionBridge.latestResult()?.utteranceId : nil)),
hostGeneration: FlowSessionBridge.currentHostGeneration()
)
)
@@ -663,6 +689,7 @@ final class FlowSessionManager: ObservableObject {
hasRecentAudio ? "audio=fresh" : "audio=stale",
isUtteranceRecording ? "recording=1" : "recording=0",
isUtteranceProcessing ? "processing=1" : "processing=0",
startingUtteranceId == nil ? "starting=0" : "starting=1",
sessionWarning == nil ? "warning=0" : "warning=1"
].joined(separator: "|")
if signature != lastReadyTraceSignature {
@@ -735,9 +762,88 @@ final class FlowSessionManager: ObservableObject {
// ASR warmup deferred to beginUtterance (first mic press).
refreshHostReady()
scheduleStartupAudioHealthCheck()
traceState("activateFlowSessionAfterPiPProof.done")
}
private func scheduleStartupAudioHealthCheck() {
guard !didRunStartupAudioHealthCheck else { return }
didRunStartupAudioHealthCheck = true
startupAudioHealthTask?.cancel()
startupAudioHealthTask = Task { @MainActor [weak self] in
guard let self else { return }
// Let deterministic Rime deployment claim the launch memory peak.
// The health probe is opportunistic and must never delay app UI.
try? await Task.sleep(nanoseconds: 300_000_000)
let heavyDeadline = Date().addingTimeInterval(8)
while FlowSessionBridge.isHostHeavy(), Date() < heavyDeadline {
guard self.isActive, !Task.isCancelled else { return }
try? await Task.sleep(nanoseconds: 200_000_000)
}
guard self.isActive,
!Task.isCancelled,
AppPermissions.flowRequirementsMet,
!FlowSessionBridge.isHostHeavy(),
self.startingUtteranceId == nil,
!self.isUtteranceRecording,
!self.isUtteranceProcessing,
!self.hasUnacknowledgedTerminalResult() else {
FlowDiagnostics.log("startup audio health check skipped")
return
}
await self.runStartupAudioHealthCheck()
}
}
private func runStartupAudioHealthCheck() async {
let healthID = UUID()
startAudioPrime(id: healthID, origin: "startupHealth")
guard let task = audioPrimeTask else { return }
_ = await task.value
guard isActive, !Task.isCancelled else { return }
// If a user utterance or touch prime took ownership, it has priority.
guard audioPrimeID == healthID,
startingUtteranceId == nil,
!isUtteranceRecording,
!isUtteranceProcessing else {
FlowDiagnostics.log("startup audio health check adopted by user")
return
}
var flowing = await capture.awaitAudioFlowing(timeout: 1.2)
if !flowing {
// One bounded rebuild exercises the same soft-dead recovery as a
// real first utterance, before the user is waiting on it.
capture.stop(releaseSession: false)
audioPrimeTask = nil
startAudioPrime(id: healthID, origin: "startupHealthRebuild")
if let rebuild = audioPrimeTask {
_ = await rebuild.value
flowing = await capture.awaitAudioFlowing(timeout: 1.0)
}
}
guard audioPrimeID == healthID,
startingUtteranceId == nil,
!isUtteranceRecording,
!isUtteranceProcessing else {
FlowDiagnostics.log("startup audio health rebuild adopted by user")
return
}
audioPrimeID = nil
audioPrimeTask = nil
audioPrimeCancellationRequested = false
if capture.running {
capture.stop(releaseSession: false)
}
_ = await pipController.reassertKeepAliveAudioSession()
refreshHostReady()
FlowDiagnostics.log(
"startup audio health check done flowing=\(flowing ? 1 : 0)"
)
}
private func prepareExistingSessionForColdStartReturn() async {
guard isColdStartHandoff, isActive else { return }
sessionWarning = nil
@@ -903,6 +1009,7 @@ final class FlowSessionManager: ObservableObject {
private func handleKeyboardSignal() {
FlowSessionBridge.reloadFromDisk()
consumeHistoryMutationOutbox()
var commands = FlowSessionBridge.commands(after: lastHandledCommandSeq)
if commands.isEmpty, let latest = FlowSessionBridge.latestCommand(),
latest.commandSeq > lastHandledCommandSeq {
@@ -922,6 +1029,20 @@ final class FlowSessionManager: ObservableObject {
consumeAckIfNeeded()
}
private func consumeHistoryMutationOutbox() {
for mutation in HistoryMutationOutbox.pending() {
let entry = SpeechHistoryStore.shared.applyHistoryMutation(mutation)
HistoryMutationReceiptStore.save(
HistoryMutationReceipt(
mutationID: mutation.id,
entryID: entry?.id,
revision: entry?.revision
)
)
HistoryMutationOutbox.acknowledge(mutation.id)
}
}
private func consumeAckIfNeeded() {
guard let ack = FlowSessionBridge.latestAck(),
let result = FlowSessionBridge.latestResult(),
@@ -954,6 +1075,43 @@ final class FlowSessionManager: ObservableObject {
|| ack.commandSeq != result.commandSeq
}
private var hostUtteranceState: FlowHostUtteranceState {
if let startingUtteranceId {
return .starting(startingUtteranceId)
}
if isUtteranceRecording, let currentUtteranceId {
return .recording(currentUtteranceId)
}
if isUtteranceProcessing, let currentUtteranceId {
return .processing(currentUtteranceId)
}
return .idle
}
private func storeRejectedStart(
_ command: FlowCommand,
message: String,
status: FlowResult.Status
) {
FlowSessionBridge.writeResult(
FlowResult(
sessionId: command.sessionId,
utteranceId: command.utteranceId,
commandSeq: command.commandSeq,
status: status,
text: message,
errorKind: .audioUnavailable,
hostGeneration: FlowSessionBridge.currentHostGeneration(),
revision: Self.resultRevision(),
utteranceMode: command.utteranceMode
)
)
traceState(
"startRecording.rejected",
extra: "status=\(status.rawValue) utterance=\(command.utteranceId.uuidString.prefix(8))"
)
}
private func handleFlowCommand(_ command: FlowCommand) {
switch FlowCommandGatekeeper.decide(
commandSessionId: command.sessionId,
@@ -982,29 +1140,83 @@ final class FlowSessionManager: ObservableObject {
lastIgnoredCommandSignature = ""
FlowDiagnostics.log(
"command \(command.action.rawValue) seq=\(command.commandSeq) utterance=\(command.utteranceId)"
"command \(command.action.rawValue) seq=\(command.commandSeq) "
+ "utterance=\(command.utteranceId) "
+ "mode=\(command.resolvedUtteranceMode.rawValue) "
+ "editSourceChars=\(command.editSourceText?.count ?? 0)"
)
switch command.action {
case .startRecording:
guard !isUtteranceRecording, !isUtteranceProcessing else { return }
guard command.resolvedUtteranceMode != .unsupportedLegacy else {
storeRejectedStart(
command,
message: AppL10n.string("flow.error.recognitionInterrupted"),
status: .error
)
traceState(
"startRecording.rejected",
extra: "reason=unsupportedLegacyMode"
)
return
}
let startDecision = FlowStartTransactionPolicy.decide(
incomingUtteranceID: command.utteranceId,
deadlineAt: command.startDeadlineAt,
hostState: hostUtteranceState
)
switch startDecision {
case .idempotent:
traceState(
"startRecording.idempotent",
extra: "utterance=\(command.utteranceId.uuidString.prefix(8))"
)
refreshHostReady()
return
case .rejectBusy:
storeRejectedStart(
command,
message: AppL10n.string("flow.error.recognitionInterrupted"),
status: .error
)
return
case .rejectExpired:
storeRejectedStart(
command,
message: AppL10n.string("flow.coldStart.error.audioTimeout"),
status: .timeout
)
return
case .accept:
break
}
guard prepareUtteranceIdentity(
utteranceId: command.utteranceId,
commandSeq: command.commandSeq
) else { return }
let deadlineAt = command.startDeadlineAt
?? Date().timeIntervalSince1970 + FlowSessionKeys.utteranceStartBudget
startingUtteranceId = command.utteranceId
startTransactionDeadlineAt = deadlineAt
FlowSessionBridge.writeStartTransaction(
FlowStartTransaction(
sessionID: command.sessionId,
utteranceID: command.utteranceId,
deadlineAt: deadlineAt,
phase: .starting
)
)
currentUtteranceMode = command.resolvedUtteranceMode
if currentUtteranceMode == .clipboardCommand {
pendingClipboardSnapshot = command.clipboardSnapshot.map {
ClipboardMaterialFilter.truncateSnapshot($0)
}
pendingPreviousOutput = command.previousOutput?
if currentUtteranceMode == .editLastInput {
let source = command.editSourceText?
.trimmingCharacters(in: .whitespacesAndNewlines)
if pendingPreviousOutput?.isEmpty == true {
pendingPreviousOutput = nil
}
pendingEditSourceText = source?.isEmpty == false ? source : nil
pendingSourceHistoryEntryID = command.sourceHistoryEntryID
pendingSourceHistoryEntryRevision = command.sourceHistoryEntryRevision
} else {
pendingClipboardSnapshot = nil
pendingPreviousOutput = nil
pendingEditSourceText = nil
pendingSourceHistoryEntryID = nil
pendingSourceHistoryEntryRevision = nil
}
guard let startUtteranceId = currentUtteranceId else { return }
let startToken = FlowUtteranceStartToken(
@@ -1015,12 +1227,18 @@ final class FlowSessionManager: ObservableObject {
await self?.handleStartRecordingCommand(
utteranceId: command.utteranceId,
commandSeq: command.commandSeq,
startToken: startToken
startToken: startToken,
deadlineAt: deadlineAt
)
}
case .stopRecording:
guard currentUtteranceId == command.utteranceId else { return }
pendingFieldContext = command.fieldContext
pendingProcessingDeadlineAt = command.processingDeadlineAt
?? (currentUtteranceMode == .editLastInput
? Date().timeIntervalSince1970
+ FlowSessionKeys.editLastInputHostProcessingBudget
: nil)
FlowDiagnostics.log(
"field context received before/after=" +
"\(command.fieldContext?.precedingText?.count ?? 0)/" +
@@ -1041,6 +1259,94 @@ final class FlowSessionManager: ObservableObject {
// No utterance identity warm SpeechAnalyzer / cloud prep only.
scheduleASRWarmup()
FlowDiagnostics.log("prewarm ASR requested seq=\(command.commandSeq)")
case .primeAudio:
beginAudioPrime(command)
case .cancelPrimeAudio:
cancelAudioPrime(command)
}
}
private func beginAudioPrime(_ command: FlowCommand) {
guard startingUtteranceId == nil,
!isUtteranceRecording,
!isUtteranceProcessing,
!hasUnacknowledgedTerminalResult() else {
return
}
startAudioPrime(id: command.utteranceId, origin: "micTouch")
}
private func startAudioPrime(id: UUID, origin: String) {
if audioPrimeTask != nil || capture.engineHasRecentAudio(maxAge: 2) {
audioPrimeID = id
audioPrimeCancellationRequested = false
return
}
audioPrimeID = id
audioPrimeCancellationRequested = false
let task = Task { @MainActor [weak self] in
guard let self else { return false }
return await self.startCaptureForPiPUtteranceIfNeeded()
}
audioPrimeTask = task
Task { @MainActor [weak self] in
let started = await task.value
guard let self else { return }
guard self.audioPrimeID == id else {
if self.audioPrimeCancellationRequested,
self.startingUtteranceId == nil,
!self.isUtteranceRecording,
!self.isUtteranceProcessing,
self.capture.running {
self.capture.stop(releaseSession: false)
_ = await self.pipController.reassertKeepAliveAudioSession()
}
self.audioPrimeTask = nil
self.audioPrimeCancellationRequested = false
self.refreshHostReady()
return
}
self.audioPrimeTask = nil
if self.audioPrimeCancellationRequested,
self.startingUtteranceId == nil,
!self.isUtteranceRecording,
!self.isUtteranceProcessing {
self.audioPrimeID = nil
self.audioPrimeCancellationRequested = false
if self.capture.running {
self.capture.stop(releaseSession: false)
_ = await self.pipController.reassertKeepAliveAudioSession()
}
self.refreshHostReady()
return
}
self.refreshHostReady()
FlowDiagnostics.log(
"audio prime completed started=\(started ? 1 : 0) "
+ "origin=\(origin) utterance=\(id.uuidString.prefix(8))"
)
}
}
private func cancelAudioPrime(_ command: FlowCommand) {
guard audioPrimeID == command.utteranceId,
startingUtteranceId == nil,
!isUtteranceRecording,
!isUtteranceProcessing else {
return
}
audioPrimeCancellationRequested = true
// `capture.start()` is not cancellation-safe. Never stop it mid-start;
// the completion waiter performs cleanup unless a real utterance adopts
// this same single-flight task first.
guard audioPrimeTask == nil else { return }
audioPrimeID = nil
audioPrimeCancellationRequested = false
capture.stop(releaseSession: false)
Task { @MainActor [weak self] in
guard let self else { return }
_ = await self.pipController.reassertKeepAliveAudioSession()
self.refreshHostReady()
}
}
@@ -1118,14 +1424,20 @@ final class FlowSessionManager: ObservableObject {
private func handleStartRecordingCommand(
utteranceId: UUID?,
commandSeq: Int64,
startToken: FlowUtteranceStartToken
startToken: FlowUtteranceStartToken,
deadlineAt: TimeInterval
) async {
guard !Task.isCancelled, canContinueStart(startToken) else { return }
guard !Task.isCancelled,
canContinueStart(startToken),
Date().timeIntervalSince1970 < deadlineAt else {
failStartIfCurrent(startToken)
return
}
refreshHostReady()
// Keep the utterance gate closed until the route is stable and the
// tap has produced a real frame. The rolling three-second preroll
// preserves speech spoken during this short readiness window.
guard await startCaptureForPiPUtteranceIfNeeded() else {
guard await ensureCaptureStartedForUtterance() else {
guard !Task.isCancelled, canContinueStart(startToken) else { return }
failUtterance(
message: AppL10n.string("flow.coldStart.error.audioTimeout"),
@@ -1137,23 +1449,32 @@ final class FlowSessionManager: ObservableObject {
releaseOrphanedCaptureIfNeeded()
return
}
guard Date().timeIntervalSince1970 < deadlineAt else {
failStartIfCurrent(startToken)
return
}
let micReady: Bool
if capture.engineHasRecentAudio(maxAge: 2) {
micReady = true
} else {
var flowing = await capture.awaitAudioFlowing(
timeout: Self.coldStartAudioProofTimeout
let firstBudget = min(
4,
max(0, deadlineAt - Date().timeIntervalSince1970)
)
if !flowing {
var flowing = await capture.awaitAudioFlowing(
timeout: firstBudget
)
let remaining = deadlineAt - Date().timeIntervalSince1970
if !flowing, remaining > 1 {
// First cold capture after PiP arm often proves audio late one rebuild.
debug("PiP audio proof timeout — one capture rebuild before failing")
capture.stop(releaseSession: false)
_ = await startCaptureForPiPUtteranceIfNeeded()
flowing = await capture.awaitAudioFlowing(
timeout: Self.coldStartAudioProofTimeout
timeout: min(2, max(0, deadlineAt - Date().timeIntervalSince1970))
)
}
micReady = flowing
micReady = flowing && Date().timeIntervalSince1970 < deadlineAt
}
guard !Task.isCancelled, canContinueStart(startToken) else {
releaseOrphanedCaptureIfNeeded()
@@ -1177,6 +1498,27 @@ final class FlowSessionManager: ObservableObject {
}
}
private func failStartIfCurrent(_ token: FlowUtteranceStartToken) {
guard canContinueStart(token) else { return }
failUtterance(
message: AppL10n.string("flow.coldStart.error.audioTimeout"),
kind: .audioUnavailable
)
}
private func ensureCaptureStartedForUtterance() async -> Bool {
audioPrimeCancellationRequested = false
audioPrimeID = nil
if let task = audioPrimeTask {
let started = await task.value
audioPrimeTask = nil
if started || capture.engineHasRecentAudio(maxAge: 2) {
return true
}
}
return await startCaptureForPiPUtteranceIfNeeded()
}
/// Start capture for a PiP utterance without blocking on the first frame.
/// Cold first start after relaunch often needs one rebuild (VPIO / -66635).
private func startCaptureForPiPUtteranceIfNeeded() async -> Bool {
@@ -1326,6 +1668,18 @@ final class FlowSessionManager: ObservableObject {
}
isUtteranceRecording = true
startingUtteranceId = nil
startTransactionDeadlineAt = nil
if let activeSessionId, let currentUtteranceId {
FlowSessionBridge.writeStartTransaction(
FlowStartTransaction(
sessionID: activeSessionId,
utteranceID: currentUtteranceId,
deadlineAt: Date().timeIntervalSince1970,
phase: .recording
)
)
}
utteranceRecordingStartedAt = Date()
startUtteranceSafetyTimer()
refreshHostReady()
@@ -1479,6 +1833,7 @@ final class FlowSessionManager: ObservableObject {
// recording flag so the poll loop cannot start a second utterance.
isUtteranceRecording = false
isUtteranceProcessing = true
FlowSessionBridge.clearStartTransaction()
utteranceSafetyTask?.cancel()
utteranceSafetyTask = nil
refreshHostReady()
@@ -1532,6 +1887,9 @@ final class FlowSessionManager: ObservableObject {
}
isUtteranceRecording = false
isUtteranceProcessing = false
startingUtteranceId = nil
startTransactionDeadlineAt = nil
FlowSessionBridge.clearStartTransaction()
utteranceRecordingStartedAt = nil
utteranceSafetyTask?.cancel()
utteranceSafetyTask = nil
@@ -1550,6 +1908,7 @@ final class FlowSessionManager: ObservableObject {
utterancePCMSamples = []
chunkWarnings = []
pendingFieldContext = nil
clearPendingInstructionState()
utteranceGeneration &+= 1
currentUtteranceId = nil
currentCommandSeq = 0
@@ -1564,6 +1923,9 @@ final class FlowSessionManager: ObservableObject {
guard claimTerminal(utteranceId: currentUtteranceId) else { return }
isUtteranceRecording = false
isUtteranceProcessing = false
startingUtteranceId = nil
startTransactionDeadlineAt = nil
FlowSessionBridge.clearStartTransaction()
utteranceRecordingStartedAt = nil
utteranceSafetyTask?.cancel()
utteranceSafetyTask = nil
@@ -1583,6 +1945,7 @@ final class FlowSessionManager: ObservableObject {
chunkWarnings = []
storeCurrentError(message, kind: kind)
pendingFieldContext = nil
clearPendingInstructionState()
utteranceGeneration &+= 1
currentUtteranceId = nil
currentCommandSeq = 0
@@ -1596,6 +1959,9 @@ final class FlowSessionManager: ObservableObject {
) {
guard claimTerminal(utteranceId: currentUtteranceId) else { return }
isUtteranceProcessing = false
startingUtteranceId = nil
startTransactionDeadlineAt = nil
FlowSessionBridge.clearStartTransaction()
utteranceRecordingStartedAt = nil
utteranceSafetyTask?.cancel()
utteranceSafetyTask = nil
@@ -1612,6 +1978,7 @@ final class FlowSessionManager: ObservableObject {
chunkWarnings = []
storeCurrentError(message, kind: kind)
pendingFieldContext = nil
clearPendingInstructionState()
utteranceGeneration &+= 1
currentUtteranceId = nil
currentCommandSeq = 0
@@ -1627,6 +1994,14 @@ final class FlowSessionManager: ObservableObject {
return true
}
private func clearPendingInstructionState() {
pendingEditSourceText = nil
pendingSourceHistoryEntryID = nil
pendingSourceHistoryEntryRevision = nil
pendingProcessingDeadlineAt = nil
currentUtteranceMode = .dictation
}
private func finalizeUtterance(
sessionId finalizeSessionId: UUID?,
utteranceId finalizeUtteranceId: UUID?,
@@ -1636,8 +2011,10 @@ final class FlowSessionManager: ObservableObject {
let pipelineStarted = Date()
let fieldContext = pendingFieldContext
let utteranceMode = currentUtteranceMode
let clipboardSnapshot = pendingClipboardSnapshot
let previousOutput = pendingPreviousOutput
let editSourceText = pendingEditSourceText
let sourceHistoryEntryID = pendingSourceHistoryEntryID
let sourceHistoryEntryRevision = pendingSourceHistoryEntryRevision
let processingDeadlineAt = pendingProcessingDeadlineAt
// ALWAYS clear the processing gate for this utterance. The previous
// guard required currentUtteranceId to still match; a racing
// fail/abort/cancel path could nil the id (or leave processing stuck)
@@ -1645,8 +2022,10 @@ final class FlowSessionManager: ObservableObject {
// while host logs still said "utterance finalized".
defer {
pendingFieldContext = nil
pendingClipboardSnapshot = nil
pendingPreviousOutput = nil
pendingEditSourceText = nil
pendingSourceHistoryEntryID = nil
pendingSourceHistoryEntryRevision = nil
pendingProcessingDeadlineAt = nil
if currentUtteranceMode == utteranceMode {
currentUtteranceMode = .dictation
}
@@ -1661,7 +2040,10 @@ final class FlowSessionManager: ObservableObject {
"finalize start asrWait=\(Int(asrWait))s engine=\(store.engineMode)"
)
let asrDeadline = Date().addingTimeInterval(asrWait)
let normalASRDeadline = Date().addingTimeInterval(asrWait)
let asrDeadline = processingDeadlineAt.map {
min(normalASRDeadline, Date(timeIntervalSince1970: $0))
} ?? normalASRDeadline
while Date() < asrDeadline {
if !lastFinal.isEmpty { break }
if asrCompletedGeneration == finalizeGeneration { break }
@@ -1708,7 +2090,12 @@ final class FlowSessionManager: ObservableObject {
+ "stitchedLen=\(lastFinal.count) partialLen=\(bestPartialSnapshot.count) "
+ "resolvedLen=\(text.count)"
)
if wantsBatchFallback, !utterancePCMSamples.isEmpty {
let hasBatchFallbackBudget = processingDeadlineAt.map {
Date().timeIntervalSince1970 + FlowSessionKeys.batchASRFallbackTimeout < $0
} ?? true
if wantsBatchFallback,
hasBatchFallbackBudget,
!utterancePCMSamples.isEmpty {
text = await runBatchASRFallback(currentText: text)
}
let textForPolish = text == lastFinal && !lastFinalWithPauseMarks.isEmpty
@@ -1750,6 +2137,9 @@ final class FlowSessionManager: ObservableObject {
commandSeq: finalizeCommandSeq
)
let recordingDuration = consumeRecordingDuration()
if utteranceMode == .editLastInput {
EditUsageMetricsStore.recordInstructionDuration(recordingDuration)
}
let engineMode = store.engineMode
let chunkNote = Self.chunkWarningMessage(chunkWarnings)
@@ -1767,35 +2157,32 @@ final class FlowSessionManager: ObservableObject {
var delivered = text
let polishStarted = Date()
let isClipboardCommand = utteranceMode == .clipboardCommand
let polishMode: PolishingService.PolishMode = isClipboardCommand
let isEditLastInput = utteranceMode == .editLastInput
let isInstructionMode = isEditLastInput
let polishMode: PolishingService.PolishMode = isInstructionMode
? .polish
: pipelineStore.polishModeForPipeline
let clipboardPrompt: (system: String, user: String)? = {
guard isClipboardCommand,
let snapshot = clipboardSnapshot,
!snapshot.isEmpty else { return nil }
let bias = ClipboardCommandPromptComposer.styleBias(
styleID: pipelineStore.activePolishStyleId,
catalog: pipelineStore.polishStyleCatalog
)
let input = ClipboardCommandPromptComposer.Input(
snapshot: snapshot,
instruction: textForPolish,
previousOutput: previousOutput,
styleBias: bias
)
return (
ClipboardCommandPromptComposer.compose(input),
ClipboardCommandPromptComposer.userMessage(input)
)
let instructionPrompt: (system: String, user: String)? = {
if isEditLastInput,
let source = editSourceText,
!source.isEmpty {
let input = EditLastInputPromptComposer.Input(
sourceText: source,
spokenInstruction: textForPolish
)
return (
EditLastInputPromptComposer.systemPrompt(),
EditLastInputPromptComposer.userMessage(input)
)
}
return nil
}()
if isClipboardCommand, clipboardPrompt == nil {
FlowDiagnostics.log("clipboard command missing snapshot — failing closed")
if isInstructionMode, instructionPrompt == nil {
FlowDiagnostics.log("instruction edit missing source — failing closed")
guard claimTerminal(utteranceId: finalizeUtteranceId) else { return }
storeFinalizedError(
AppL10n.string("flow.error.clipboardCommandFailed"),
AppL10n.string("flow.error.editLastInputFailed"),
kind: .generic,
sessionId: finalizeSessionId,
utteranceId: finalizeUtteranceId,
@@ -1804,14 +2191,16 @@ final class FlowSessionManager: ObservableObject {
return
}
let modeLabel = isEditLastInput
? "editLastInput"
: Self.polishModeLogLabel(polishMode)
FlowDiagnostics.log(
"finalize LLM mode=\(isClipboardCommand ? "clipboardCommand" : Self.polishModeLogLabel(polishMode)) " +
"translationTarget=\(pipelineStore.translationTargetLocaleId)"
"finalize LLM mode=\(modeLabel) translationTarget=\(pipelineStore.translationTargetLocaleId)"
)
FlowTrace.transcript(
"polish.input",
textForPolish,
"mode=\(isClipboardCommand ? "clipboardCommand" : Self.polishModeLogLabel(polishMode)) engine=\(engineMode) "
"mode=\(modeLabel) engine=\(engineMode) "
+ "provider=\(pipelineStore.polishProviderIdOverride ?? "default") "
+ "recordedSeconds=\(String(format: "%.2f", recordingDuration))"
)
@@ -1819,31 +2208,54 @@ final class FlowSessionManager: ObservableObject {
// If the finalize task was cancelled (cold-start churn / abort),
// skip the LLM round-trip and deliver the raw transcript so the
// keyboard is not left waiting on a result that never arrives.
// Clipboard-command mode must never insert the instruction ASR.
// Edit mode must never insert the instruction ASR.
if Task.isCancelled {
throw CancellationError()
}
let outcome = try await Self.polishWithHostTimeout(
polisher: polisher,
text: clipboardPrompt?.user ?? textForPolish,
text: instructionPrompt?.user ?? textForPolish,
mode: polishMode,
systemPrompt: clipboardPrompt?.system,
systemPrompt: instructionPrompt?.system,
providerIdOverride: pipelineStore.polishProviderIdOverride,
context: isClipboardCommand ? nil : polishContext
context: isInstructionMode ? nil : polishContext,
timeoutLimit: processingDeadlineAt.map {
max(0.1, $0 - Date().timeIntervalSince1970)
}
)
let polished = outcome.text
let polished: String
if isEditLastInput, let source = editSourceText {
switch EditOutputValidator.validate(sourceText: source, output: outcome.text) {
case .success(let validated):
polished = validated
case .failure(let validationError):
throw validationError
}
} else {
polished = outcome.text
}
delivered = polished
FlowTrace.transcript(
"polish.output",
polished,
"mode=\(isClipboardCommand ? "clipboardCommand" : Self.polishModeLogLabel(polishMode)) inputLen=\(text.count) "
"mode=\(modeLabel) inputLen=\(text.count) "
+ "changed=\(polished == text ? 0 : 1) "
+ "elapsed=\(FlowTrace.seconds(since: polishStarted))s"
)
guard claimTerminal(utteranceId: finalizeUtteranceId) else { return }
let historyEntry = isEditLastInput
? nil
: SpeechHistoryStore.shared.recordUtterance(
text: delivered,
engineMode: engineMode,
duration: recordingDuration,
wasTranslation: isInstructionMode
? false
: pipelineStore.isTranslationEffective
)
storeFinalizedResult(
polished,
rawText: isClipboardCommand ? nil : text,
rawText: isInstructionMode ? nil : text,
warning: Self.combinedWarning(
chunkNote,
outcome.qualityDegraded
@@ -1852,33 +2264,33 @@ final class FlowSessionManager: ObservableObject {
),
sessionId: finalizeSessionId,
utteranceId: finalizeUtteranceId,
commandSeq: finalizeCommandSeq
commandSeq: finalizeCommandSeq,
historyEntryID: isEditLastInput
? sourceHistoryEntryID
: historyEntry?.id,
historyEntryRevision: isEditLastInput
? sourceHistoryEntryRevision
: historyEntry?.revision
)
FlowDiagnostics.log(
"polish done in \(String(format: "%.1f", Date().timeIntervalSince(polishStarted)))s " +
"total=\(String(format: "%.1f", Date().timeIntervalSince(pipelineStarted)))s"
)
SpeechHistoryStore.shared.recordUtterance(
text: delivered,
engineMode: engineMode,
duration: recordingDuration,
wasTranslation: isClipboardCommand ? false : pipelineStore.isTranslationEffective
)
} catch {
if isClipboardCommand {
if isInstructionMode {
FlowDiagnostics.log(
"clipboard command failed after \(String(format: "%.1f", Date().timeIntervalSince(polishStarted)))s: " +
"instruction edit failed after \(String(format: "%.1f", Date().timeIntervalSince(polishStarted)))s: " +
"\(error.localizedDescription)"
)
FlowTrace.warn(
"clipboardCommand.failed",
"editLastInput.failed",
"elapsed=\(FlowTrace.seconds(since: polishStarted))s "
+ "cancelled=\(error is CancellationError ? 1 : 0) "
+ "error=\(error.localizedDescription)"
)
guard claimTerminal(utteranceId: finalizeUtteranceId) else { return }
storeFinalizedError(
AppL10n.string("flow.error.clipboardCommandFailed"),
AppL10n.string("flow.error.editLastInputFailed"),
kind: .generic,
sessionId: finalizeSessionId,
utteranceId: finalizeUtteranceId,
@@ -1912,19 +2324,21 @@ final class FlowSessionManager: ObservableObject {
)
delivered = fallback.text
guard claimTerminal(utteranceId: finalizeUtteranceId) else { return }
let historyEntry = SpeechHistoryStore.shared.recordUtterance(
text: delivered,
engineMode: engineMode,
duration: recordingDuration,
wasTranslation: pipelineStore.isTranslationEffective
)
storeFinalizedResult(
fallback.text,
rawText: text,
warning: fallback.polishWarning,
sessionId: finalizeSessionId,
utteranceId: finalizeUtteranceId,
commandSeq: finalizeCommandSeq
)
SpeechHistoryStore.shared.recordUtterance(
text: delivered,
engineMode: engineMode,
duration: recordingDuration,
wasTranslation: pipelineStore.isTranslationEffective
commandSeq: finalizeCommandSeq,
historyEntryID: historyEntry?.id,
historyEntryRevision: historyEntry?.revision
)
}
}
@@ -1975,7 +2389,9 @@ final class FlowSessionManager: ObservableObject {
warning: String?,
sessionId: UUID?,
utteranceId: UUID?,
commandSeq: Int64
commandSeq: Int64,
historyEntryID: UUID? = nil,
historyEntryRevision: Int64? = nil
) {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else {
@@ -2007,7 +2423,9 @@ final class FlowSessionManager: ObservableObject {
hostGeneration: FlowSessionBridge.currentHostGeneration(),
revision: Self.resultRevision(),
fieldFingerprint: Self.fieldFingerprint(pendingFieldContext),
utteranceMode: currentUtteranceMode
utteranceMode: currentUtteranceMode,
historyEntryID: historyEntryID,
historyEntryRevision: historyEntryRevision
)
)
}
@@ -2067,8 +2485,12 @@ final class FlowSessionManager: ObservableObject {
)
}
private static var lastResultRevision: Int64 = 0
private static func resultRevision() -> Int64 {
Int64(Date().timeIntervalSince1970 * 1_000)
let millis = Int64(Date().timeIntervalSince1970 * 1_000)
lastResultRevision = max(lastResultRevision + 1, millis)
return lastResultRevision
}
private static func fieldFingerprint(_ context: FlowFieldContext?) -> String? {
@@ -2198,9 +2620,11 @@ final class FlowSessionManager: ObservableObject {
mode: PolishingService.PolishMode,
systemPrompt: String? = nil,
providerIdOverride: String?,
context: PolishContext?
context: PolishContext?,
timeoutLimit: TimeInterval? = nil
) async throws -> PolishingService.PolishOutcome {
let timeout = FlowSessionKeys.polishTimeout(forCharacterCount: text.count)
let scaled = FlowSessionKeys.polishTimeout(forCharacterCount: text.count)
let timeout = timeoutLimit.map { min(scaled, $0) } ?? scaled
return try await HardTimeout.run(seconds: timeout) {
try await polisher.polishWithOutcome(
text,
@@ -0,0 +1,93 @@
// RimeDeploymentController.swift
// OSGKeyboard · Main App
//
// Single entry point for user-visible Rime deployment.
//
// Chinese typing is unusable until Rime is deployed, so this controller runs
// host-owned deployment immediately when resources are missing and exposes an
// observable outcome to onboarding and Settings. The keyboard extension only
// reads readiness and opens the already-built data.
import Foundation
import Combine
import OSGKeyboardShared
@MainActor
final class RimeDeploymentController: ObservableObject {
static let shared = RimeDeploymentController()
enum Status: Equatable {
case idle
case deploying
case ready
case failed(String)
}
@Published private(set) var status: Status = .idle
private var activeTask: Task<Void, Never>?
var isDeploying: Bool { status == .deploying }
init() {
status = RimeResourceInstaller.isReady ? .ready : .idle
}
/// Re-reads App Group state, e.g. after another process deployed.
func refreshStatus() {
guard !isDeploying else { return }
if RimeResourceInstaller.isReady {
status = .ready
} else if case .failed = status {
// Keep the failure visible until a retry actually runs.
} else {
status = .idle
}
}
/// Deploys right away, deliberately skipping the warmup delay, foreground
/// check, and memory gate. Callers are moments where the user is waiting on
/// the result and cannot be racing the keyboard extension for memory.
func deployNow(force: Bool = false, reason: String) {
guard activeTask == nil else { return }
if !force, RimeResourceInstaller.isReady {
status = .ready
return
}
OSGDiag.log("rime.deployNow begin reason=\(reason) \(OSGDiag.memoryTag())", category: "flow")
status = .deploying
let snapshot = TypingInputConfiguration.shared.snapshot
activeTask = Task { @MainActor in
defer { activeTask = nil }
// Mirrors the warmup path so the keyboard defers its own prepare
// while librime maintenance is running.
FlowSessionBridge.setHostHeavy(true)
do {
try await RimeResourceInstaller.shared.installIfNeeded(
configuration: snapshot,
force: force
)
// Release the heavy-work gate before notifying the extension.
// Its observer retries immediately and must not see stale busy
// state, or the error remains until the keyboard is reopened.
FlowSessionBridge.setHostHeavy(false)
AppGroupConfigDarwin.postConfigChanged()
status = .ready
OSGDiag.log(
"rime.deployNow done reason=\(reason) \(OSGDiag.memoryTag())",
category: "flow"
)
} catch {
FlowSessionBridge.setHostHeavy(false)
status = .failed(error.localizedDescription)
OSGDiag.log(
"rime.deployNow failed reason=\(reason) error=\(error.localizedDescription)",
category: "flow"
)
}
}
}
}
@@ -8,17 +8,19 @@ import OSGKeyboardShared
extension SpeechHistoryStore {
/// Append history and update cumulative home-screen usage stats.
@discardableResult
func recordUtterance(
text: String,
engineMode: String,
duration: TimeInterval,
wasTranslation: Bool
) {
append(text: text, engineMode: engineMode)
) -> SpeechHistoryEntry? {
let entry = append(text: text, engineMode: engineMode)
UsageStatisticsStore.shared.recordUtterance(
text: text,
duration: duration,
wasTranslation: wasTranslation
)
return entry
}
}
+259
View File
@@ -0,0 +1,259 @@
// EditDemoView.swift
// OSGKeyboard · Main App (DEBUG-only)
//
// Scripted, keyboard-sized recreation of the extension's `LastInputEditView`
// used ONLY to record the "Edit last input" What's New clip in the simulator.
// It reuses the real shared `EditTextPager`, design tokens, and the real
// `EditSessionState` machine, and steps a fixed timeline (idle hint listening
// processing review swipe apply) with canned text no ASR, no LLM.
// Launched via `--edit-demo` (see OSGKeyboardApp). Not shipped in Release.
#if DEBUG
import SwiftUI
import OSGKeyboardShared
struct EditDemoView: View {
// Canned material for the clip.
private static let originalText = "明天下午三点开会讨论方案"
private static let editedText = "各位好,明天下午三点在 A 会议室召开方案讨论会,请准时参加。"
private let palette = Palette.light
@State private var editSession: EditSessionState = .inactive
@State private var showHint = true
@State private var selectedPage: Int? = 0
@State private var remainingSeconds = 59
private var source: EditSessionSource {
let reference = EditableInputReference(
displayText: Self.originalText,
insertedText: Self.originalText,
postInsertionFingerprint: nil,
extensionInstanceID: UUID()
)
return EditSessionSource(reference: reference)
}
var body: some View {
ZStack {
Color(red: 0.06, green: 0.06, blue: 0.07).ignoresSafeArea()
VStack(spacing: 0) {
Spacer(minLength: 0)
keyboardPanel
.background(panelBackground)
.overlay(alignment: .top) {
Rectangle()
.fill(palette.divider)
.frame(height: 0.5)
}
}
}
.environment(\.themePalette, palette)
.task { await runTimeline() }
}
private var panelBackground: some View {
palette.background.ignoresSafeArea(edges: .bottom)
}
// MARK: - Panel (mirrors LastInputEditView layout)
private var keyboardPanel: some View {
VStack(spacing: 0) {
topBar.frame(height: 44)
if showHint {
hintBody
} else {
pages.frame(height: 144)
statusLine.frame(height: 18)
pageIndicator.frame(height: 12)
primaryRow.frame(height: 55)
}
}
.padding(.vertical, 4)
.padding(.horizontal, KeyboardChromeLayout.horizontalInset)
.frame(maxWidth: .infinity)
.frame(height: KeyboardChromeLayout.totalHeight)
.padding(.bottom, 24)
}
private var topBar: some View {
HStack {
Text("OSG")
.font(.system(size: 17, weight: .heavy, design: .rounded))
.foregroundStyle(palette.accent)
Spacer(minLength: 0)
Image(systemName: "xmark")
.font(.system(size: 19, weight: .semibold))
.foregroundStyle(palette.textPrimary)
.frame(width: 44, height: 44)
.background(palette.surfaceElevated.opacity(0.72), in: Circle())
}
// keyboardPanel already contributes 8pt; add the nested 4pt so the
// effective top-bar inset matches the normal voice surface's 12pt.
.padding(.horizontal, Spacing.xs)
}
// Opening frame: idle mic + "" hint.
private var hintBody: some View {
VStack(spacing: Spacing.sm) {
Spacer(minLength: 0)
Text("长按可编辑上一条")
.font(TypeStyle.footnote)
.foregroundStyle(palette.accent)
ZStack {
Circle().fill(palette.accent)
Image(systemName: "mic.fill")
.font(.system(size: 21, weight: .semibold))
.foregroundStyle(.white)
}
.frame(width: 64, height: 64)
.shadow(color: palette.accentGlow, radius: 12)
Spacer(minLength: 0)
}
.frame(maxWidth: .infinity, maxHeight: .infinity)
}
@ViewBuilder
private var pages: some View {
EditTextPager(
originalTitle: "原文",
originalText: Self.originalText,
editedTitle: "编辑后",
editedText: editSession.review?.resultText,
selectedPage: $selectedPage
)
}
private var statusLine: some View {
Text(statusText)
.font(TypeStyle.caption2)
.foregroundStyle(palette.textSecondary)
.lineLimit(1)
.frame(maxWidth: .infinity)
}
private var pageIndicator: some View {
HStack(spacing: 5) {
Circle()
.fill(selectedPage != 1 ? palette.textPrimary : palette.textTertiary.opacity(0.45))
.frame(width: 5, height: 5)
Circle()
.fill(selectedPage == 1 ? palette.textPrimary : palette.textTertiary.opacity(0.45))
.frame(width: 5, height: 5)
}
.opacity(editSession.review == nil ? 0 : 1)
}
private var primaryRow: some View {
HStack(spacing: Spacing.sm) {
helperText(leftHelper)
ZStack {
Capsule().fill(palette.accent)
primaryIcon
}
.frame(width: 150, height: 50)
helperText(rightHelper)
}
}
private func helperText(_ value: String) -> some View {
Text(value)
.font(TypeStyle.caption2)
.foregroundStyle(palette.textSecondary.opacity(0.55))
.multilineTextAlignment(.center)
.lineLimit(2)
.minimumScaleFactor(0.75)
.frame(maxWidth: .infinity)
}
@ViewBuilder
private var primaryIcon: some View {
switch editSession {
case .processing, .applying, .appending:
ProgressView().tint(.white)
case .review:
Image(systemName: "checkmark")
.font(.system(size: 21, weight: .bold))
.foregroundStyle(.white)
case .listening:
VStack(spacing: 0) {
Text(formatRemaining(remainingSeconds))
.font(.system(size: 10, weight: .semibold, design: .rounded))
.monospacedDigit()
Image(systemName: "mic.fill")
.font(.system(size: 16, weight: .semibold))
}
.foregroundStyle(.white)
default:
Image(systemName: "mic.fill")
.font(.system(size: 21, weight: .semibold))
.foregroundStyle(.white)
}
}
// MARK: - Copy (mirrors ExtL10n zh keyboard.edit.*)
private var statusText: String {
switch editSession {
case .listening: return "正在聆听编辑指令"
case .processing: return "正在编辑…"
case .review: return "左右滑动对比原文和结果"
case .applying, .appending: return "正在应用编辑…"
default: return ""
}
}
private var leftHelper: String {
editSession.review == nil ? "说话编辑文字" : "左右滑动对比"
}
private var rightHelper: String {
editSession.review != nil ? "点击应用编辑" : "点击完成编辑"
}
private func formatRemaining(_ seconds: Int) -> String {
"\(seconds / 60):\(String(format: "%02d", seconds % 60))"
}
// MARK: - Scripted timeline
private func runTimeline() async {
let src = source
let review = EditReview(source: src, resultText: Self.editedText, utteranceID: UUID())
try? await sleep(1.3) // idle hint
withAnimation(.easeInOut(duration: 0.25)) {
showHint = false
editSession = .listening(src)
}
// Tick the utterance countdown while listening.
for _ in 0..<3 {
try? await sleep(0.5)
remainingSeconds -= 1
}
withAnimation(.easeInOut(duration: 0.2)) { editSession = .processing(src) }
try? await sleep(1.3)
withAnimation(.easeInOut(duration: 0.25)) {
editSession = .review(review)
selectedPage = 0
}
try? await sleep(1.1)
withAnimation(.interactiveSpring(response: 0.34, dampingFraction: 0.86)) {
selectedPage = 1
}
try? await sleep(1.6)
withAnimation(.easeInOut(duration: 0.2)) { editSession = .applying(review) }
try? await sleep(0.9)
}
private func sleep(_ seconds: Double) async throws {
try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000))
}
}
#endif
+8 -2
View File
@@ -162,12 +162,18 @@ struct HomeView: View {
VStack(alignment: .leading, spacing: Spacing.lg) {
wideHeroHeader
HomeUsageStatsSection(layout: .split)
// On iPad / regular width the keyboard-setup hint (and any
// other flow-session extras) used to render below
// `HomeUsageStatsSection`, burying the most actionable guidance
// beneath the stats cards. Match the phone layout's ordering:
// hero header hint stats preview, so the hint sits at
// the top of the page and is the first thing a user notices.
if showsFlowSessionExtras {
flowSessionExtras
}
HomeUsageStatsSection(layout: .split)
widePreviewStage
}
.padding(.horizontal, WideLayoutMetrics.pageHorizontalInset)
+76 -38
View File
@@ -17,17 +17,11 @@ struct MainAppRoot: View {
@ObservedObject private var config = ProviderConfig.shared
@ObservedObject private var releaseNotes = ReleaseNotesController.shared
@StateObject private var flowManager = FlowSessionManager()
@State private var postOnboardingWarmupTask: Task<Void, Never>?
@State private var clmWarmupTask: Task<Void, Never>?
var body: some View {
Group {
if config.hasCompletedOnboarding {
MainTabView()
.id("main")
} else {
OnboardingView(config: config)
.id("onboarding")
}
mainContent
}
.environment(\.locale, config.uiLanguage.swiftUILocale)
.environmentObject(flowManager)
@@ -71,10 +65,14 @@ struct MainAppRoot: View {
// Heavy work (Flow / CLM / Rime) only after onboarding. Doing it
// earlier jetsams the host (~150 MB+) and the keyboard dies with it.
if config.hasCompletedOnboarding {
// Rime deployment is host-only and idempotent. Run it
// immediately when missing so returning users never have to
// wait for an opportunistic background warmup.
RimeDeploymentController.shared.deployNow(reason: "MainAppRoot.onAppear")
// Automatically arm the low-profile PiP on every host open.
// Capture/ASR remain lazy and start only on an actual mic press.
flowManager.activateOnForeground(reason: "MainAppRoot.onAppear")
schedulePostOnboardingWarmup(reason: "MainAppRoot.onAppear")
scheduleCLMWarmup(reason: "MainAppRoot.onAppear")
releaseNotes.presentIfNeeded(onboardingCompleted: true)
} else {
OSGDiag.log(
@@ -89,22 +87,28 @@ struct MainAppRoot: View {
.onChange(of: config.hasCompletedOnboarding) { _, done in
if done {
flowManager.activateOnForeground(reason: "onboardingCompleted")
schedulePostOnboardingWarmup(reason: "onboardingCompleted")
// Deploy now rather than via warmup: the user just finished
// setup, is still in the app, and has not started using the
// keyboard yet so there is nothing to race for memory. This
// also covers users who skipped the keyboard page entirely.
RimeDeploymentController.shared.deployNow(reason: "onboardingCompleted")
scheduleCLMWarmup(reason: "onboardingCompleted")
releaseNotes.presentIfNeeded(onboardingCompleted: true)
}
}
.onChange(of: scenePhase) { _, phase in
flowManager.handleScenePhase(phase)
guard phase == .active else {
postOnboardingWarmupTask?.cancel()
postOnboardingWarmupTask = nil
clmWarmupTask?.cancel()
clmWarmupTask = nil
FlowSessionBridge.setHostHeavy(false)
return
}
if config.hasCompletedOnboarding {
RimeDeploymentController.shared.deployNow(reason: "scenePhase.active")
flowManager.activateOnForeground(reason: "scenePhase.active")
// Retry deferred Rime/CLM after a jetsam-prone launch.
schedulePostOnboardingWarmup(reason: "scenePhase.active.retry")
// Retry deferred CLM after a jetsam-prone launch.
scheduleCLMWarmup(reason: "scenePhase.active.retry")
releaseNotes.presentIfNeeded(onboardingCompleted: true)
}
Task {
@@ -113,32 +117,43 @@ struct MainAppRoot: View {
}
}
/// Serial host warmup: Rime deploy CLM. Never parallel with ASR.
@ViewBuilder
private var mainContent: some View {
if config.hasCompletedOnboarding {
MainTabView()
.id("main")
} else {
OnboardingView(config: config)
.id("onboarding")
}
}
/// Delayed host CLM warmup. Never parallel with ASR.
/// ASR warms on first mic press (`FlowSessionManager.beginUtterance`).
///
/// Intentionally delayed: running Rime/CLM on `onAppear` kept the host at
/// ~175 MB while the user switched to the keyboard, and the extension died
/// before `KVC.init` (no dyld breadcrumb).
private func schedulePostOnboardingWarmup(reason: String) {
/// Intentionally delayed: warming CLM on `onAppear` kept the host at
/// ~175 MB while the user switched to the keyboard. Rime is excluded from
/// this opportunistic path because missing typing resources block users.
private func scheduleCLMWarmup(reason: String) {
OSGDiag.log(
"postOnboardingWarmup scheduled reason=\(reason) delay=45s \(OSGDiag.memoryTag())",
"clmWarmup scheduled reason=\(reason) delay=45s \(OSGDiag.memoryTag())",
category: "flow"
)
postOnboardingWarmupTask?.cancel()
postOnboardingWarmupTask = Task { @MainActor in
clmWarmupTask?.cancel()
clmWarmupTask = Task { @MainActor in
// Let the user leave the host / cold-start the keyboard first.
try? await Task.sleep(nanoseconds: 45_000_000_000)
guard !Task.isCancelled else { return }
guard scenePhase == .active else {
OSGDiag.log(
"postOnboardingWarmup skip reason=notActive \(OSGDiag.memoryTag())",
"clmWarmup skip reason=notActive \(OSGDiag.memoryTag())",
category: "flow"
)
return
}
guard !flowManager.shouldDeferHostHeavyWork else {
OSGDiag.log(
"postOnboardingWarmup skip reason=flowBusy \(OSGDiag.memoryTag())",
"clmWarmup skip reason=flowBusy \(OSGDiag.memoryTag())",
category: "flow"
)
return
@@ -146,20 +161,6 @@ struct MainAppRoot: View {
await AppCloudSync.shared.pullAllIfEnabled()
// hostHeavy only while heavy work runs never leave it stuck at 1
// just because RSS is above the soft gate (that blocked typing).
guard HostMemoryBudget.gate("rime.installIfNeeded") else { return }
FlowSessionBridge.setHostHeavy(true)
defer { FlowSessionBridge.setHostHeavy(false) }
let typingConfig = TypingInputConfiguration.shared.snapshot
OSGDiag.log("rime.installIfNeeded begin \(OSGDiag.memoryTag())", category: "flow")
try? await RimeResourceInstaller.shared.installIfNeeded(
configuration: typingConfig
)
OSGDiag.log("rime.installIfNeeded done \(OSGDiag.memoryTag())", category: "flow")
guard HostMemoryBudget.gate("clm.prepare", category: "asr") else { return }
CustomLanguageModelManager.shared.prepareInBackgroundIfNeeded()
OSGDiag.log("clm.prepare scheduled \(OSGDiag.memoryTag())", category: "asr")
@@ -171,6 +172,10 @@ struct MainAppRoot: View {
switch url.host {
case "startflow":
flowManager.startSession(coldStart: true, reason: "url.startflow")
case "deployrime":
// The keyboard sends the user here precisely because typing
// resources are missing deploy without waiting for warmup.
RimeDeploymentController.shared.deployNow(reason: "url.deployrime")
#if DEBUG
case "seed-demo":
DemoDataSeeder.seedRichPlaceholderData()
@@ -181,3 +186,36 @@ struct MainAppRoot: View {
}
}
}
#if DEBUG
struct EditPagerUITestHarness: View {
@State private var selectedPage: Int? = 0
var body: some View {
VStack(spacing: 20) {
ZStack(alignment: .bottom) {
EditTextPager(
originalTitle: "Original",
originalText: "ORIGINAL_PAGE_TOKEN",
editedTitle: "Edited",
editedText: "EDITED_PAGE_TOKEN",
contentBottomInset: 30,
selectedPage: $selectedPage
)
.frame(maxWidth: .infinity, maxHeight: .infinity)
Color.clear
.frame(height: 30)
.allowsHitTesting(false)
}
.frame(height: 220)
.accessibilityIdentifier("edit.pager.swipeArea")
Text(selectedPage == 1 ? "EDITED_ACTIVE" : "ORIGINAL_ACTIVE")
.accessibilityIdentifier("edit.pager.activePage")
}
.padding()
.environment(\.themePalette, Palette.light)
}
}
#endif
+31 -4
View File
@@ -70,7 +70,10 @@ struct MainSplitView: View {
.padding(.leading, WideLayoutMetrics.sidebarContentInset)
.padding(.trailing, WideLayoutMetrics.sidebarInset)
.padding(.top, Spacing.lg)
.padding(.bottom, Spacing.md)
// Roughly double the gap between the brand header and the first
// sidebar row on iPad so the logo gets visual breathing room above
// the menu list (was `Spacing.md`, now `Spacing.xxxl`).
.padding(.bottom, Spacing.xxxl)
}
private var devicesFooter: some View {
@@ -108,14 +111,19 @@ private struct WideSidebarRow: View {
var body: some View {
Button(action: action) {
Label(tab.sidebarTitle, systemImage: tab.sidebarSystemImage)
.font(.system(size: 13, weight: isSelected ? .semibold : .regular))
.labelStyle(SidebarIconColumnLabelStyle())
// Slightly larger label + taller vertical padding so each
// sidebar row hits the Apple HIG 44pt touch target on iPad
// (was size 13 / vertical 7 ~32pt row; now size 15 /
// vertical 12 ~44pt row).
.font(.system(size: 15, weight: isSelected ? .semibold : .regular))
.foregroundStyle(isSelected ? palette.accent : palette.textPrimary)
.frame(maxWidth: .infinity, alignment: .leading)
.padding(.horizontal, Spacing.sm)
.padding(.vertical, 7)
.padding(.vertical, 12)
.background(
rowBackground,
in: RoundedRectangle(cornerRadius: 7, style: .continuous)
in: RoundedRectangle(cornerRadius: Radius.medium, style: .continuous)
)
.contentShape(Rectangle())
}
@@ -129,6 +137,25 @@ private struct WideSidebarRow: View {
}
}
/// Reserves a uniform icon column so every sidebar title starts at the same x.
///
/// `DefaultLabelStyle` sizes the icon to each SF Symbol's intrinsic width, and
/// this sidebar's symbols range from ~13pt (`character.book.closed`) to ~17.5pt
/// (`house`) enough to push titles apart by ~5pt. `List` reserves that column
/// automatically; this hand-rolled `VStack` sidebar has to do it itself.
private struct SidebarIconColumnLabelStyle: LabelStyle {
/// Wider than the widest symbol in `AppTab.sidebarSystemImage`.
private static let iconColumnWidth: CGFloat = 22
func makeBody(configuration: Configuration) -> some View {
HStack(spacing: Spacing.xs) {
configuration.icon
.frame(width: Self.iconColumnWidth)
configuration.title
}
}
}
// MARK: - Status footer
/// Quiet bottom strip: engine mode + translation target + Flow readiness.
+46
View File
@@ -631,6 +631,7 @@ private struct PermissionPageLayout: View {
private struct EnableKeyboardPage: View {
@Environment(\.themePalette) private var palette: ThemePalette
@ObservedObject private var deployment = RimeDeploymentController.shared
var body: some View {
ScrollView {
@@ -671,9 +672,54 @@ private struct EnableKeyboardPage: View {
.padding(.horizontal, Spacing.lg)
.padding(.top, Spacing.xxxl)
resourceStatusRow
.padding(.horizontal, Spacing.lg)
.padding(.top, Spacing.md)
Spacer(minLength: Spacing.lg)
}
}
// Deploy while the user reads these steps and visits system settings.
// The keyboard is not in use yet, so this is the one moment where the
// host can afford the memory without risking the extension.
.onAppear {
deployment.deployNow(reason: "onboarding.enableKeyboard")
}
}
@ViewBuilder
private var resourceStatusRow: some View {
switch deployment.status {
case .deploying:
HStack(spacing: Spacing.xs) {
ProgressView().controlSize(.small)
Text(LocalizedStringKey("onboarding.enable.resources.preparing"))
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
}
case .ready:
HStack(spacing: Spacing.xs) {
Image(systemName: "checkmark.circle.fill")
.foregroundStyle(palette.accent)
Text(LocalizedStringKey("onboarding.enable.resources.ready"))
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
}
case .failed:
HStack(spacing: Spacing.xs) {
Image(systemName: "exclamationmark.triangle.fill")
.foregroundStyle(palette.warning)
Text(LocalizedStringKey("onboarding.enable.resources.failed"))
.font(TypeStyle.caption)
.foregroundStyle(palette.textSecondary)
Button(LocalizedStringKey("onboarding.enable.resources.retry")) {
deployment.deployNow(force: true, reason: "onboarding.retry")
}
.font(TypeStyle.caption)
}
case .idle:
EmptyView()
}
}
private func step(num: Int, text: String) -> some View {
+10 -19
View File
@@ -11,9 +11,9 @@ struct TypingInputSettingsView: View {
@Environment(\.themePalette) private var palette: ThemePalette
@ObservedObject private var config = ProviderConfig.shared
@ObservedObject private var configuration = TypingInputConfiguration.shared
@ObservedObject private var deployment = RimeDeploymentController.shared
@State private var isDeploying = false
@State private var deploymentError: String?
private var isDeploying: Bool { deployment.isDeploying }
var body: some View {
List {
@@ -59,7 +59,7 @@ struct TypingInputSettingsView: View {
} else {
Text(statusText)
.foregroundStyle(
deploymentError == nil ? palette.textSecondary : palette.danger
hasDeploymentError ? palette.danger : palette.textSecondary
)
}
}
@@ -76,8 +76,13 @@ struct TypingInputSettingsView: View {
.navigationBarTitleDisplayMode(.inline)
}
private var hasDeploymentError: Bool {
if case .failed = deployment.status { return true }
return false
}
private var statusText: String {
if let deploymentError { return deploymentError }
if case .failed(let message) = deployment.status { return message }
return AppL10n.string(
RimeResourceInstaller.isReady
? "settings.typingInput.resources.ready"
@@ -87,20 +92,6 @@ struct TypingInputSettingsView: View {
}
private func deployUpdatedSchemas() {
guard !isDeploying else { return }
let snapshot = configuration.snapshot
isDeploying = true
deploymentError = nil
Task {
do {
try await RimeResourceInstaller.shared.installIfNeeded(
configuration: snapshot,
force: true
)
} catch {
deploymentError = error.localizedDescription
}
isDeploying = false
}
deployment.deployNow(force: true, reason: "settings.typingInput")
}
}
+5 -1
View File
@@ -27,6 +27,10 @@
"onboarding.enable.step3.prefix" = "Hold ";
"onboarding.enable.step3.suffix" = "and select OSGKeyboard";
"onboarding.enable.openSettings" = "Open Settings";
"onboarding.enable.resources.preparing" = "Preparing Chinese input resources…";
"onboarding.enable.resources.ready" = "Chinese input resources ready";
"onboarding.enable.resources.failed" = "Could not prepare Chinese input resources";
"onboarding.enable.resources.retry" = "Retry";
"onboarding.api.title" = "Choose Engine";
"onboarding.api.localModels.hint" = "Local engine uses built-in on-device speech recognition — no API key needed.";
"onboarding.polish.title" = "Text polish (LLM)";
@@ -352,7 +356,6 @@
/* Flow session */
"flow.error.noSpeech" = "No speech detected. Please try again.";
"flow.error.clipboardCommandFailed" = "Couldn't process the clipboard. Please try again.";
"flow.error.recognitionInterrupted" = "Recognition did not finish. Please try again.";
"keyboard.denied.mic" = "Microphone access denied";
"keyboard.denied.speech" = "Speech recognition denied";
@@ -557,3 +560,4 @@
"hostApp.douyin" = "Douyin";
"hostApp.tiktok" = "TikTok";
"flow.warning.polishDegradedQuality" = "Polish output failed validation; inserted a conservative version.";
"flow.error.editLastInputFailed" = "Could not complete the edit. Please try again.";
@@ -27,6 +27,10 @@
"onboarding.enable.step3.prefix" = "长按";
"onboarding.enable.step3.suffix" = ",选中 OSGKeyboard";
"onboarding.enable.openSettings" = "去设置";
"onboarding.enable.resources.preparing" = "正在准备中文输入资源…";
"onboarding.enable.resources.ready" = "中文输入资源已就绪";
"onboarding.enable.resources.failed" = "中文输入资源准备失败";
"onboarding.enable.resources.retry" = "重试";
"onboarding.api.title" = "选择语音转文字 AI 引擎";
"onboarding.api.localModels.hint" = "本地引擎使用内置语音识别,无需填写 API Key。";
"onboarding.polish.title" = "文本润色(LLM";
@@ -351,7 +355,6 @@
/* Flow session */
"flow.error.noSpeech" = "未检测到语音,请重试。";
"flow.error.clipboardCommandFailed" = "剪贴板处理失败,请重试。";
"flow.error.recognitionInterrupted" = "识别未完成,请再试一次。";
"keyboard.denied.mic" = "麦克风权限被拒绝";
"keyboard.denied.speech" = "语音识别权限被拒绝";
@@ -556,3 +559,4 @@
"hostApp.douyin" = "抖音";
"hostApp.tiktok" = "TikTok";
"flow.warning.polishDegradedQuality" = "本次润色结果未通过校验,已插入保守处理版本。";
"flow.error.editLastInputFailed" = "未能完成编辑,请重试。";