feat(macos): local ASR model manager, menu-bar polish, and release 0.5.2

Adds a bundled local ASR model catalog for the macOS app with one-click
Sherpa Qwen3 / SenseVoice downloads (pause/resume, inline actions) and a
shared model storage directory used by MLX Qwen3. Fixes the light-mode
sidebar material and makes the menu-bar icon follow the system appearance
with a refreshed status mark. Renames the built product to OSGKeyboard.app.

Bumps version to 0.5.2 (build 19).
This commit is contained in:
Rocky
2026-07-09 08:55:37 +08:00
parent c2f07bd8d2
commit 200265fbd6
50 changed files with 4666 additions and 266 deletions
@@ -0,0 +1,25 @@
{
"images" : [
{
"filename" : "OSGStatusMark.png",
"idiom" : "universal",
"scale" : "1x"
},
{
"idiom" : "universal",
"scale" : "2x"
},
{
"idiom" : "universal",
"scale" : "3x"
}
],
"info" : {
"author" : "xcode",
"version" : 1
},
"properties" : {
"preserves-vector-representation" : true,
"template-rendering-intent" : "template"
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 3.9 KiB

@@ -10,12 +10,16 @@ import OSGKeyboardShared
enum FlowLiveActivityController {
nonisolated(unsafe) private static var currentActivity: Activity<FlowActivityAttributes>?
/// Last phase pushed to the Live Activity so `keepAlive()` can refresh the
/// `staleDate` without changing what the user sees.
nonisolated(unsafe) private static var currentPhase: FlowActivityAttributes.ContentState.Phase = .idle
/// If the host app is force-quit its `endSession()` never runs, orphaning
/// the Live Activity. A `staleDate` lets the system grey it out and become
/// willing to reclaim it without our process refreshed on every update
/// so a genuinely active, in-use session never looks stale.
private static let staleWindow: TimeInterval = 60 * 60
/// the Live Activity. A short `staleDate` lets the system grey it out and
/// reclaim it on its own within ~45s of the process dying. While the host is
/// alive the heartbeat calls `keepAlive()` well inside this window, so a
/// genuinely active session never looks stale.
private static let staleWindow: TimeInterval = 45
private static func freshContent(
phase: FlowActivityAttributes.ContentState.Phase
@@ -41,6 +45,7 @@ enum FlowLiveActivityController {
}
do {
currentPhase = .idle
currentActivity = try Activity.request(
attributes: FlowActivityAttributes(),
content: freshContent(phase: .idle),
@@ -54,14 +59,28 @@ enum FlowLiveActivityController {
static func update(phase: FlowActivityAttributes.ContentState.Phase) {
guard let activity = currentActivity else { return }
currentPhase = phase
let content = freshContent(phase: phase)
Task {
await activity.update(content)
}
}
/// Push a fresh `staleDate` without changing the visible phase. The host
/// heartbeat calls this well inside `staleWindow` so an in-use session
/// never looks stale; once the process dies the refreshes stop and the
/// system reclaims the orphaned Live Activity on its own.
static func keepAlive() {
guard let activity = currentActivity else { return }
let content = freshContent(phase: currentPhase)
Task {
await activity.update(content)
}
}
/// Dismiss the island presentation when the Flow session ends.
static func endSession() {
currentPhase = .idle
guard let activity = currentActivity else {
endStaleActivities()
return
@@ -74,6 +93,18 @@ enum FlowLiveActivityController {
}
}
/// Clear Live Activities orphaned by a previous (force-quit) host process.
///
/// Safe to call on every app foreground: when this process already owns a
/// Live Activity (`currentActivity != nil`) we leave it alone so a healthy
/// running session is never torn down; we only sweep leftovers that belong
/// to a dead process. Call this *before* attempting to (re)start a session
/// so a failed start (e.g. mic timeout) still clears the stale island.
static func clearOrphanedActivities() {
guard currentActivity == nil else { return }
endStaleActivities()
}
/// Host relaunch can leave orphan activities; clear them before starting anew.
private static func endStaleActivities() {
let staleActivities = Activity<FlowActivityAttributes>.activities
+407 -75
View File
@@ -25,9 +25,7 @@ final class FlowSessionManager: ObservableObject {
private let capture = FlowContinuousCapture()
private let store = AppGroupStore()
/// Cloud-engine polish; local engine runs through built-in DeepSeek polish.
private var polisher: PolishingService {
PolishingService()
}
private let polisher = PolishingService()
/// Cached ASR instance. v0.2.0: the only on-device backend is iOS
/// `SpeechAnalyzer`, which has no warm-up step we can hand the
/// factory-built service straight back without going through the
@@ -49,17 +47,26 @@ final class FlowSessionManager: ObservableObject {
private var expiryTask: Task<Void, Never>?
private var levelTask: Task<Void, Never>?
private var startTask: Task<Void, Never>?
private var commandObserver: FlowSessionDarwinObserver?
/// Last recording state the poll loop observed logs only on transition.
private var lastObservedRecordingState: FlowSessionKeys.RecordingState = .idle
private var activeSessionId: UUID?
private var currentUtteranceId: UUID?
private var currentCommandSeq: Int64 = 0
private var lastHandledCommandSeq: Int64 = 0
private var isUtteranceRecording = false
/// True from `stopped` until the result/error is written back to App Group.
private var isUtteranceProcessing = false
private var finalizeTask: Task<Void, Never>?
private var asrTask: Task<Void, Never>?
private var utteranceSafetyTask: Task<Void, Never>?
private var chunkedPipeline: ChunkedUtterancePipeline?
private var currentPartial = ""
private var lastFinal = ""
private var chunkWarnings: [String] = []
private var lastReadyTraceSignature = ""
private var lastCommandFingerprint = ""
private var lastIgnoredCommandSignature = ""
/// Wall-clock span of the current mic-open utterance (excludes LLM polish).
private var utteranceRecordingStartedAt: Date?
/// True while the host app scene is `.active` drives foreground renewal.
@@ -67,7 +74,11 @@ final class FlowSessionManager: ObservableObject {
private var backgroundTaskID: UIBackgroundTaskIdentifier = .invalid
/// True while handling a keyboard-initiated `startflow` cold start.
private var isColdStartHandoff = false
private static let coldStartAudioProofTimeout: TimeInterval = 2.5
private var coldStartRecoveryTask: Task<Void, Never>?
/// Initial proof window cold mic sessions often need >2.5s after app switch.
private static let coldStartAudioProofTimeout: TimeInterval = 6
/// Extra window after the first timeout while the overlay shows a failure hint.
private static let coldStartRecoveryProofTimeout: TimeInterval = 12
init() {
// Sessions are (re)started explicitly on app foreground via
@@ -83,6 +94,10 @@ final class FlowSessionManager: ObservableObject {
/// Starts a Flow session: permissions continuous capture App Group active.
func startSession(duration: TimeInterval? = nil, coldStart: Bool = false) {
traceState(
"startSession.request",
extra: "coldStart=\(coldStart) duration=\(Int(duration ?? FlowSessionPolicy.sessionDuration()))"
)
guard AppGroup.isAvailable else {
debug("cannot start flow session: App Group unavailable")
return
@@ -106,7 +121,10 @@ final class FlowSessionManager: ObservableObject {
return
}
guard !isStarting else { return }
guard !isStarting else {
traceState("startSession.ignored", extra: "reason=alreadyStarting")
return
}
startTask?.cancel()
startTask = Task { @MainActor [weak self] in
@@ -148,6 +166,14 @@ final class FlowSessionManager: ObservableObject {
/// behind (its `endSession()` could not run at kill time).
func activateOnForeground() {
guard AppGroup.isAvailable else { return }
// Sweep any Live Activity a previous (force-quit) process left behind
// *before* we try to (re)start a session. Doing it here rather than
// only inside `startSession()`'s success path means a start that
// later fails (e.g. mic proof timeout) still clears the stale island
// instead of leaving a zombie on the lock screen / Dynamic Island.
// No-op when this process already owns a healthy Live Activity.
FlowLiveActivityController.clearOrphanedActivities()
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
FlowSessionBridge.setHostReady(false)
@@ -161,7 +187,8 @@ final class FlowSessionManager: ObservableObject {
}
func dismissColdStartOverlay() {
guard coldStartContext?.state != .preparing else { return }
coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = nil
if isActive {
refreshHostReady()
}
@@ -189,8 +216,11 @@ final class FlowSessionManager: ObservableObject {
coldStartContext = nil
isColdStartHandoff = false
coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = nil
startTask?.cancel()
startTask = nil
commandObserver = nil
pollingTask?.cancel()
pollingTask = nil
heartbeatTask?.cancel()
@@ -201,6 +231,8 @@ final class FlowSessionManager: ObservableObject {
levelTask = nil
finalizeTask?.cancel()
finalizeTask = nil
utteranceSafetyTask?.cancel()
utteranceSafetyTask = nil
if isUtteranceRecording || isUtteranceProcessing {
capture.cancelUtterance()
@@ -210,6 +242,10 @@ final class FlowSessionManager: ObservableObject {
}
asrTask = nil
chunkedPipeline = nil
activeSessionId = nil
currentUtteranceId = nil
currentCommandSeq = 0
lastHandledCommandSeq = 0
isUtteranceRecording = false
isUtteranceProcessing = false
@@ -305,9 +341,11 @@ final class FlowSessionManager: ObservableObject {
guard isActive else { return }
if capture.running {
capture.reassertIfRunning()
if capture.engineHasRecentAudio() {
let reasserted = capture.reassertIfRunning()
if reasserted, capture.engineHasRecentAudio() {
sessionWarning = nil
} else if !reasserted {
sessionWarning = AppL10n.string("flow.error.audioUnavailable")
}
refreshHostReady()
return
@@ -329,22 +367,94 @@ final class FlowSessionManager: ObservableObject {
/// Publish whether the keyboard can start a new utterance without jumping to the host app.
private func refreshHostReady() {
guard isActive else {
FlowSessionBridge.setHostReady(false)
FlowSessionBridge.writeReadySnapshot(
FlowReadySnapshot(
sessionId: activeSessionId,
ready: false,
reason: .noSession,
engineMode: store.engineMode,
localeId: store.localeId,
sessionExpiresAt: FlowSessionBridge.sessionExpiresAt()
)
)
return
}
let pollingAlive = pollingTask != nil && pollingTask?.isCancelled != true
// Steady-state ready uses structural engine liveness. The stricter
// "recent audio frame" proof is reserved for cold-start handoff only
// (`waitForAudioProof`) so brief UI-driven session hiccups do not
// drop the keyboard back to orange while the host app is foreground.
let hasRecentAudio = capture.engineHasRecentAudio(maxAge: 2)
let canAcceptUtterance = capture.engineIsLive
&& pollingAlive
&& hasRecentAudio
&& !isUtteranceRecording
&& !isUtteranceProcessing
&& sessionWarning == nil
FlowSessionBridge.setHostReady(canAcceptUtterance)
let reason: FlowReadySnapshot.Reason
if canAcceptUtterance {
reason = .ready
} else if sessionWarning != nil {
reason = .error
} else if isUtteranceRecording {
reason = .recording
} else if isUtteranceProcessing {
reason = .processing
} else if !capture.engineIsLive {
reason = .audioEngineNotLive
} else if !hasRecentAudio {
reason = .waitingForAudioProof
} else {
reason = .starting
}
let now = Date().timeIntervalSince1970
FlowSessionBridge.writeReadySnapshot(
FlowReadySnapshot(
sessionId: activeSessionId,
ready: canAcceptUtterance,
reason: reason,
heartbeatAt: now,
readyAt: canAcceptUtterance ? now : nil,
audioProofAt: hasRecentAudio ? now : nil,
engineMode: store.engineMode,
localeId: store.localeId,
busyUtteranceId: isUtteranceRecording || isUtteranceProcessing ? currentUtteranceId : nil,
sessionExpiresAt: FlowSessionBridge.sessionExpiresAt()
)
)
let signature = [
canAcceptUtterance ? "ready=1" : "ready=0",
"reason=\(reason.rawValue)",
capture.engineIsLive ? "engine=live" : "engine=dead",
hasRecentAudio ? "audio=fresh" : "audio=stale",
isUtteranceRecording ? "recording=1" : "recording=0",
isUtteranceProcessing ? "processing=1" : "processing=0",
sessionWarning == nil ? "warning=0" : "warning=1"
].joined(separator: "|")
if signature != lastReadyTraceSignature {
lastReadyTraceSignature = signature
traceState("hostReady.update", extra: signature)
}
reconcileColdStartOverlayIfRecovered()
}
/// When the host contract turns green while the cold-start overlay still
/// shows a stale preparing/failed snapshot, heal automatically.
private func reconcileColdStartOverlayIfRecovered() {
guard isColdStartHandoff, isActive else { return }
guard FlowSessionBridge.isHostReady() else { return }
guard let context = coldStartContext else { return }
switch context.state {
case .preparing:
presentColdStartReadyOverlay()
case .failed:
sessionWarning = nil
coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = nil
dismissColdStartOverlay()
case .ready:
break
}
}
/// Home preview field gained focus while this app is the Flow host.
@@ -378,12 +488,14 @@ final class FlowSessionManager: ObservableObject {
// MARK: - Session start
private func startSessionAsync(duration: TimeInterval?) async {
traceState("startSessionAsync.begin")
isStarting = true
sessionWarning = nil
defer { isStarting = false }
guard AppPermissions.flowRequirementsMet else {
sessionWarning = permissionWarningMessage()
traceState("startSessionAsync.blocked", extra: "reason=permissions")
FlowSessionBridge.setHostReady(false)
if isColdStartHandoff {
showColdStartPermissionFailure()
@@ -397,6 +509,7 @@ final class FlowSessionManager: ObservableObject {
} catch {
let message = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
sessionWarning = message
traceState("startSessionAsync.failed", extra: "reason=captureStart error=\(message)")
FlowSessionBridge.setHostReady(false)
if isColdStartHandoff {
showColdStartAudioFailure(message: message)
@@ -408,23 +521,36 @@ final class FlowSessionManager: ObservableObject {
guard await waitForAudioProof() else {
let message = AppL10n.string("flow.coldStart.error.audioTimeout")
sessionWarning = message
capture.stop()
traceState("startSessionAsync.failed", extra: "reason=audioProofTimeout")
FlowSessionBridge.setHostReady(false)
if isColdStartHandoff {
showColdStartAudioFailure(message: message)
scheduleColdStartRecovery(duration: duration)
} else {
capture.stop()
}
debug("continuous capture did not produce audio frames before timeout")
return
}
activateFlowSessionAfterAudioProof(duration: duration)
traceState("startSessionAsync.ready")
debug("Flow session started (\(Int(duration ?? FlowSessionPolicy.sessionDuration()))s inactivity window), continuous capture running")
}
private func activateFlowSessionAfterAudioProof(duration: TimeInterval?) {
let resolvedDuration = duration ?? FlowSessionPolicy.sessionDuration()
FlowSessionBridge.markSessionActive(duration: resolvedDuration)
let sessionId = activeSessionId ?? UUID()
activeSessionId = sessionId
lastHandledCommandSeq = 0
FlowSessionBridge.markSessionActive(duration: resolvedDuration, sessionId: sessionId)
FlowSessionDarwin.postSessionChanged()
isActive = true
ScreenWakeLock.acquire()
sessionExpiresAt = Date().addingTimeInterval(resolvedDuration)
startHeartbeat()
startCommandObserver()
startPolling()
startLevelPublishing()
scheduleExpiry(after: resolvedDuration)
@@ -434,7 +560,7 @@ final class FlowSessionManager: ObservableObject {
FlowLiveActivityController.startSession()
refreshHostReady()
debug("Flow session started (\(Int(resolvedDuration))s inactivity window), continuous capture running")
traceState("activateFlowSessionAfterAudioProof.done")
}
private func prepareExistingSessionForColdStartReturn() async {
@@ -445,6 +571,7 @@ final class FlowSessionManager: ObservableObject {
sessionWarning = message
FlowSessionBridge.setHostReady(false)
showColdStartAudioFailure(message: message)
scheduleColdStartRecovery(duration: nil)
debug("existing session failed cold-start audio proof")
return
}
@@ -466,25 +593,54 @@ final class FlowSessionManager: ObservableObject {
let message = AppL10n.string("flow.coldStart.error.audioTimeout")
sessionWarning = message
showColdStartAudioFailure(message: message)
scheduleColdStartRecovery(duration: nil)
debug("cold-start blocked: host ready contract not published")
return
}
let hostEntry = HostReturnService.pendingHostEntry()
let skipSwitch = FlowSessionPolicy.skipAppSwitch()
coldStartContext = FlowColdStartContext(hostEntry: hostEntry, state: .ready)
presentColdStartReadyOverlay()
}
if skipSwitch, hostEntry != nil {
Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: 450_000_000)
guard let self, self.coldStartContext?.state == .ready else { return }
if HostReturnService.openPendingHostIfPossible() {
self.dismissColdStartOverlay()
}
private func presentColdStartReadyOverlay() {
let hostEntry = HostReturnService.pendingHostEntry()
coldStartContext = FlowColdStartContext(hostEntry: hostEntry, state: .ready)
scheduleAutoReturnToHostIfNeeded(hostEntry: hostEntry)
}
private func scheduleAutoReturnToHostIfNeeded(hostEntry: HostAppEntry?) {
let skipSwitch = FlowSessionPolicy.skipAppSwitch()
guard skipSwitch, hostEntry != nil else { return }
Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: 450_000_000)
guard let self, self.coldStartContext?.state == .ready else { return }
if HostReturnService.openPendingHostIfPossible() {
self.dismissColdStartOverlay()
}
}
}
/// Keeps proving mic readiness after the first timeout instead of tearing
/// capture down many handoffs become ready a few seconds later.
private func scheduleColdStartRecovery(duration: TimeInterval?) {
coldStartRecoveryTask?.cancel()
coldStartRecoveryTask = Task { @MainActor [weak self] in
guard let self else { return }
let recovered = await self.capture.awaitAudioFlowing(
timeout: Self.coldStartRecoveryProofTimeout
)
guard !Task.isCancelled else { return }
guard self.isColdStartHandoff else { return }
guard recovered else { return }
self.sessionWarning = nil
self.traceState("coldStartRecovery.recovered")
if !self.isActive {
self.activateFlowSessionAfterAudioProof(duration: duration)
}
self.refreshHostReady()
}
}
private func showColdStartPreparing() {
coldStartContext = FlowColdStartContext(
hostEntry: HostReturnService.pendingHostEntry(),
@@ -546,6 +702,14 @@ final class FlowSessionManager: ObservableObject {
// MARK: - Polling
private func startCommandObserver() {
commandObserver = FlowSessionDarwinObserver(
notificationName: FlowSessionDarwin.commandNotificationName
) { [weak self] in
self?.handleKeyboardSignal()
}
}
private func startPolling() {
pollingTask?.cancel()
lastObservedRecordingState = FlowSessionBridge.recordingState()
@@ -556,46 +720,122 @@ final class FlowSessionManager: ObservableObject {
pollingTask = Task { @MainActor [weak self] in
while !Task.isCancelled {
self?.handleKeyboardSignal()
try? await Task.sleep(nanoseconds: 50_000_000)
try? await Task.sleep(nanoseconds: 500_000_000)
}
}
}
private func handleKeyboardSignal() {
let signal = FlowSessionBridge.recordingState()
if signal != lastObservedRecordingState {
// The single most important cross-process signal: proves whether the
// host actually SEES the keyboard's recording state writes.
FlowDiagnostics.log(
"poll observed recordingState \(lastObservedRecordingState.rawValue)\(signal.rawValue) " +
"[rec=\(isUtteranceRecording) proc=\(isUtteranceProcessing) fg=\(isAppForeground)]"
)
lastObservedRecordingState = signal
guard let command = FlowSessionBridge.latestCommand() else {
lastCommandFingerprint = ""
return
}
switch signal {
case .recording:
let fingerprint = "\(command.sessionId.uuidString)|\(command.utteranceId.uuidString)|\(command.action.rawValue)|\(command.commandSeq)"
guard fingerprint != lastCommandFingerprint else { return }
lastCommandFingerprint = fingerprint
handleFlowCommand(command)
}
private func handleFlowCommand(_ command: FlowCommand) {
guard let activeSessionId, command.sessionId == activeSessionId else {
traceIgnoredCommand(
reason: "staleSession",
command: command,
detail: "commandSession=\(command.sessionId)"
)
return
}
guard command.commandSeq > lastHandledCommandSeq else {
traceIgnoredCommand(
reason: "seqNotIncreasing",
command: command,
detail: "last=\(lastHandledCommandSeq)"
)
return
}
lastHandledCommandSeq = command.commandSeq
lastIgnoredCommandSignature = ""
FlowDiagnostics.log(
"command \(command.action.rawValue) seq=\(command.commandSeq) utterance=\(command.utteranceId)"
)
switch command.action {
case .startRecording:
guard !isUtteranceRecording, !isUtteranceProcessing else { return }
beginUtterance()
case .stopped:
beginUtterance(utteranceId: command.utteranceId, commandSeq: command.commandSeq)
case .stopRecording:
guard currentUtteranceId == command.utteranceId else { return }
if isUtteranceRecording {
endUtterance()
} else if !isUtteranceProcessing {
FlowSessionBridge.setRecordingState(.idle)
FlowSessionBridge.storeTranscriptionError(
storeCurrentError(
AppL10n.string("flow.error.recognitionInterrupted"),
kind: .recognitionInterrupted
)
debug("stopped without active utterance — notified keyboard")
debug("stop command without active utterance — notified keyboard")
}
case .aborted:
case .abort:
guard currentUtteranceId == command.utteranceId else { return }
abortUtterance()
case .idle, .processing:
break
}
}
private func beginUtterance() {
guard capture.engineIsLive else {
private func storeCurrentPartial(_ text: String) {
guard let activeSessionId, let currentUtteranceId else { return }
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
FlowSessionBridge.writeResult(
FlowResult(
sessionId: activeSessionId,
utteranceId: currentUtteranceId,
commandSeq: currentCommandSeq,
status: .partial,
text: trimmed
)
)
}
private func storeCurrentFinal(_ text: String, warning: String? = nil) {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else {
storeCurrentError(AppL10n.string("flow.error.noSpeech"), kind: .noSpeech)
return
}
guard let activeSessionId, let currentUtteranceId else { return }
FlowSessionBridge.writeResult(
FlowResult(
sessionId: activeSessionId,
utteranceId: currentUtteranceId,
commandSeq: currentCommandSeq,
status: .final,
text: trimmed,
warning: warning
)
)
}
private func storeCurrentError(
_ message: String,
kind: FlowSessionKeys.TranscriptionErrorKind = .generic,
status: FlowResult.Status = .error
) {
guard let activeSessionId, let currentUtteranceId else { return }
FlowSessionBridge.writeResult(
FlowResult(
sessionId: activeSessionId,
utteranceId: currentUtteranceId,
commandSeq: currentCommandSeq,
status: status,
text: message,
errorKind: kind
)
)
}
private func beginUtterance(utteranceId: UUID? = nil, commandSeq: Int64 = 0) {
guard capture.engineHasRecentAudio(maxAge: 2) else {
traceState("beginUtterance.blocked", extra: "reason=audioNotRecent")
failUtterance(
message: AppL10n.string("flow.error.audioUnavailable"),
kind: .audioUnavailable
@@ -603,13 +843,25 @@ final class FlowSessionManager: ObservableObject {
return
}
guard !isUtteranceProcessing else {
traceState("beginUtterance.ignored", extra: "reason=processing")
debug("beginUtterance ignored — previous utterance still processing")
return
}
bindSessionASRIfNeeded()
let expectedEngine = store.engineMode
if sessionASREngineMode != expectedEngine {
traceState(
"beginUtterance.rebindMismatch",
extra: "expectedEngine=\(expectedEngine) boundEngine=\(sessionASREngineMode ?? "nil")"
)
bindSessionASRIfNeeded(force: true)
}
// Usually already warm from session start; refresh without blocking the mic gate.
scheduleASRWarmup()
currentUtteranceId = utteranceId ?? UUID()
currentCommandSeq = commandSeq
currentPartial = ""
lastFinal = ""
chunkWarnings = []
@@ -617,6 +869,7 @@ final class FlowSessionManager: ObservableObject {
let localeId = store.localeId
FlowSessionBridge.setTranscriptionLanguage(localeId)
FlowSessionBridge.clearPendingTranscription()
FlowSessionBridge.clearResult()
let locale = SpeechLocaleResolver.resolve(localeId)
let stream = capture.beginUtterance()
@@ -625,6 +878,7 @@ final class FlowSessionManager: ObservableObject {
isUtteranceRecording = true
utteranceRecordingStartedAt = Date()
startUtteranceSafetyTimer()
refreshHostReady()
FlowLiveActivityController.update(phase: .recording)
FlowDiagnostics.log(
@@ -637,8 +891,9 @@ final class FlowSessionManager: ObservableObject {
asrTask = Task.detached(priority: .userInitiated) { [weak manager = self] in
let outcome = await pipeline.transcribe(stream: stream) { partial in
Task { @MainActor in
manager?.currentPartial = partial
FlowSessionBridge.storeTranscriptionPartial(partial)
guard let manager else { return }
manager.currentPartial = partial
manager.storeCurrentPartial(partial)
}
}
// Re-bind `manager` inside the `@MainActor` block so the
@@ -672,14 +927,33 @@ final class FlowSessionManager: ObservableObject {
debug("utterance recording started")
}
private func startUtteranceSafetyTimer() {
utteranceSafetyTask?.cancel()
let utteranceId = currentUtteranceId
utteranceSafetyTask = Task { @MainActor [weak self] in
let timeout = FlowSessionKeys.maxUtteranceDuration + 10
try? await Task.sleep(nanoseconds: UInt64(timeout * 1_000_000_000))
guard let self, !Task.isCancelled else { return }
guard self.isUtteranceRecording, self.currentUtteranceId == utteranceId else { return }
self.storeCurrentError(
AppL10n.string("flow.error.recognitionInterrupted"),
kind: .recognitionInterrupted,
status: .timeout
)
self.abortUtterance()
self.debug("utterance safety timer aborted stale recording")
}
}
private func endUtterance() {
guard isUtteranceRecording else { return }
// Close the mic gate first, then mark processing before dropping the
// recording flag so the poll loop cannot start a second utterance.
FlowSessionBridge.setRecordingState(.processing)
isUtteranceRecording = false
isUtteranceProcessing = true
utteranceSafetyTask?.cancel()
utteranceSafetyTask = nil
refreshHostReady()
FlowLiveActivityController.update(phase: .processing)
@@ -699,6 +973,8 @@ final class FlowSessionManager: ObservableObject {
isUtteranceRecording = false
isUtteranceProcessing = false
utteranceRecordingStartedAt = nil
utteranceSafetyTask?.cancel()
utteranceSafetyTask = nil
finalizeTask?.cancel()
finalizeTask = nil
asrTask?.cancel()
@@ -709,8 +985,8 @@ final class FlowSessionManager: ObservableObject {
currentPartial = ""
lastFinal = ""
chunkWarnings = []
FlowSessionBridge.storeTranscriptionPartial("")
FlowSessionBridge.setRecordingState(.idle)
currentUtteranceId = nil
currentCommandSeq = 0
FlowLiveActivityController.update(phase: .idle)
refreshHostReady()
debug("utterance aborted")
@@ -723,6 +999,8 @@ final class FlowSessionManager: ObservableObject {
isUtteranceRecording = false
isUtteranceProcessing = false
utteranceRecordingStartedAt = nil
utteranceSafetyTask?.cancel()
utteranceSafetyTask = nil
finalizeTask?.cancel()
finalizeTask = nil
asrTask?.cancel()
@@ -733,9 +1011,9 @@ final class FlowSessionManager: ObservableObject {
currentPartial = ""
lastFinal = ""
chunkWarnings = []
FlowSessionBridge.storeTranscriptionPartial("")
FlowSessionBridge.storeTranscriptionError(message, kind: kind)
FlowSessionBridge.setRecordingState(.idle)
storeCurrentError(message, kind: kind)
currentUtteranceId = nil
currentCommandSeq = 0
FlowLiveActivityController.update(phase: .idle)
refreshHostReady()
debug("utterance failed: \(message)")
@@ -747,28 +1025,36 @@ final class FlowSessionManager: ObservableObject {
) {
isUtteranceProcessing = false
utteranceRecordingStartedAt = nil
utteranceSafetyTask?.cancel()
utteranceSafetyTask = nil
finalizeTask?.cancel()
finalizeTask = nil
chunkedPipeline = nil
currentPartial = ""
lastFinal = ""
chunkWarnings = []
FlowSessionBridge.storeTranscriptionPartial("")
FlowSessionBridge.storeTranscriptionError(message, kind: kind)
FlowSessionBridge.setRecordingState(.idle)
storeCurrentError(message, kind: kind)
currentUtteranceId = nil
currentCommandSeq = 0
FlowLiveActivityController.update(phase: .idle)
refreshHostReady()
debug("utterance processing failed: \(message)")
}
private func finalizeUtterance() async {
let finalizeSessionId = activeSessionId
let finalizeUtteranceId = currentUtteranceId
let pipelineStarted = Date()
defer {
isUtteranceProcessing = false
FlowSessionBridge.setRecordingState(.idle)
FlowLiveActivityController.update(phase: .idle)
touchSessionActivity()
refreshHostReady()
if activeSessionId == finalizeSessionId,
currentUtteranceId == finalizeUtteranceId {
isUtteranceProcessing = false
FlowLiveActivityController.update(phase: .idle)
touchSessionActivity()
currentUtteranceId = nil
currentCommandSeq = 0
refreshHostReady()
}
}
let asrWait = asrWaitTimeout()
@@ -783,9 +1069,10 @@ final class FlowSessionManager: ObservableObject {
try? await Task.sleep(nanoseconds: 100_000_000)
}
if lastFinal.isEmpty, let asrTask {
FlowDiagnostics.log("ASR wait elapsed — awaiting asrTask completion")
_ = await asrTask.value
if lastFinal.isEmpty {
FlowDiagnostics.log("ASR wait elapsed — cancelling ASR task and using best available transcript")
asrTask?.cancel()
Task { await chunkedPipeline?.cancel() }
}
let asrElapsed = Date().timeIntervalSince(pipelineStarted)
@@ -803,10 +1090,7 @@ final class FlowSessionManager: ObservableObject {
(asrTask?.isCancelled == true) ? .recognitionInterrupted : .noSpeech
FlowDiagnostics.log("finalize failed: empty transcript after \(String(format: "%.1f", asrElapsed))s")
utteranceRecordingStartedAt = nil
FlowSessionBridge.storeTranscriptionError(
AppL10n.string(key),
kind: kind
)
storeCurrentError(AppL10n.string(key), kind: kind)
return
}
@@ -832,7 +1116,7 @@ final class FlowSessionManager: ObservableObject {
providerIdOverride: pipelineStore.polishProviderIdOverride
)
delivered = polished
FlowSessionBridge.storeTranscriptionResult(polished, polishWarning: chunkNote)
storeCurrentFinal(polished, warning: chunkNote)
FlowDiagnostics.log(
"polish done in \(String(format: "%.1f", Date().timeIntervalSince(polishStarted)))s " +
"total=\(String(format: "%.1f", Date().timeIntervalSince(pipelineStarted)))s"
@@ -854,7 +1138,7 @@ final class FlowSessionManager: ObservableObject {
"\(error.localizedDescription)"
)
delivered = fallback.text
FlowSessionBridge.storeTranscriptionResult(fallback.text, polishWarning: fallback.polishWarning)
storeCurrentFinal(fallback.text, warning: fallback.polishWarning)
}
SpeechHistoryStore.shared.recordUtterance(
@@ -867,7 +1151,6 @@ final class FlowSessionManager: ObservableObject {
currentPartial = ""
lastFinal = ""
chunkWarnings = []
FlowSessionBridge.storeTranscriptionPartial("")
chunkedPipeline = nil
debug("utterance finalized length=\(text.count)")
}
@@ -964,6 +1247,12 @@ final class FlowSessionManager: ObservableObject {
heartbeatTask?.cancel()
FlowSessionBridge.writeHeartbeat()
heartbeatTask = Task { @MainActor [weak self] in
// Refresh the Live Activity `staleDate` every N heartbeat ticks
// (1 Hz) well inside `FlowLiveActivityController.staleWindow` so a
// live session never looks stale, while a force-quit stops these
// refreshes and lets the system reclaim the orphaned island.
let liveActivityKeepAliveEveryTicks = 15
var tick = 0
while !Task.isCancelled {
guard let self else { break }
if self.isActive, !self.capture.engineIsLive {
@@ -971,6 +1260,10 @@ final class FlowSessionManager: ObservableObject {
}
FlowSessionBridge.writeHeartbeat()
self.refreshHostReady()
tick += 1
if tick % liveActivityKeepAliveEveryTicks == 0 {
FlowLiveActivityController.keepAlive()
}
try? await Task.sleep(nanoseconds: 1_000_000_000)
guard self.isActive else { break }
}
@@ -989,4 +1282,43 @@ final class FlowSessionManager: ObservableObject {
private func debug(_ message: String) {
FlowDiagnostics.log(message)
}
private func traceIgnoredCommand(reason: String, command: FlowCommand, detail: String) {
let signature = "\(reason)|\(command.action.rawValue)|\(command.commandSeq)|\(command.sessionId.uuidString)|\(command.utteranceId.uuidString)|\(detail)"
guard signature != lastIgnoredCommandSignature else { return }
lastIgnoredCommandSignature = signature
traceState(
"command.ignored",
extra: "reason=\(reason) action=\(command.action.rawValue) seq=\(command.commandSeq) \(detail)"
)
}
private func traceState(_ event: String, extra: String? = nil) {
let staleness = FlowSessionBridge.heartbeatStaleness().map { String(format: "%.1f", $0) } ?? "nil"
let sessionId = activeSessionId?.uuidString ?? "nil"
let utteranceId = currentUtteranceId?.uuidString ?? "nil"
let summary = [
"event=\(event)",
"active=\(isActive)",
"starting=\(isStarting)",
"coldStart=\(isColdStartHandoff)",
"sessionId=\(sessionId)",
"utteranceId=\(utteranceId)",
"cmdSeq=\(currentCommandSeq)",
"lastCmd=\(lastHandledCommandSeq)",
"recording=\(isUtteranceRecording)",
"processing=\(isUtteranceProcessing)",
"storeEngine=\(store.engineMode)",
"boundEngine=\(sessionASREngineMode ?? "nil")",
"engineLive=\(capture.engineIsLive)",
"hostReady=\(FlowSessionBridge.isHostReady())",
"sessionActive=\(FlowSessionBridge.isSessionActive())",
"heartbeatStaleness=\(staleness)"
].joined(separator: " ")
if let extra, !extra.isEmpty {
debug("[trace] \(summary) \(extra)")
} else {
debug("[trace] \(summary)")
}
}
}
+21 -8
View File
@@ -38,6 +38,17 @@ struct FlowColdStartOverlay: View {
/// Fraction of the screen height the bottom gradient occupies.
private let gradientHeightFraction: CGFloat = 0.50
/// Ready and failure states dismiss on blank tap; preparing stays
/// informational only (no accidental dismiss while proving audio).
private var allowsBlankTapDismiss: Bool {
switch context.state {
case .ready, .failed:
return true
case .preparing:
return false
}
}
var body: some View {
GeometryReader { geo in
ZStack(alignment: .bottom) {
@@ -56,6 +67,16 @@ struct FlowColdStartOverlay: View {
.frame(maxWidth: .infinity, alignment: .bottom)
.allowsHitTesting(false)
if allowsBlankTapDismiss {
// Captures taps on empty overlay space (dismiss) and blocks
// pass-through to the host shell underneath. Action buttons in
// `content` sit above this layer and remain tappable.
Color.clear
.contentShape(Rectangle())
.onTapGesture(perform: onDismiss)
.ignoresSafeArea()
}
VStack(spacing: Spacing.lg) {
content
.padding(.horizontal, Spacing.xl)
@@ -63,14 +84,6 @@ struct FlowColdStartOverlay: View {
homeIndicator
.padding(.bottom, max(geo.safeAreaInsets.bottom, Spacing.sm))
}
.allowsHitTesting(false)
if context.state == .ready {
Color.clear
.contentShape(Rectangle())
.onTapGesture(perform: onDismiss)
.ignoresSafeArea()
}
}
.frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .bottom)
.ignoresSafeArea()