feat: add streaming cloud ASR, polish routing, and settings card layout

Unify Bailian/Volcengine/OpenAI realtime streaming, ABE polish routing with
fun styles, and a shared card-page Settings hierarchy; bump to 1.1 (build 32).
This commit is contained in:
Rocky
2026-07-28 20:25:17 +08:00
parent 956331a2af
commit d656bac8c3
58 changed files with 4153 additions and 1396 deletions
+107 -25
View File
@@ -543,9 +543,15 @@ final class FlowSessionManager: ObservableObject {
private func reactivateCaptureIfNeeded() async {
guard isActive else { return }
if usesPiPKeepAlive, !isUtteranceRecording, !isUtteranceProcessing, !capture.running {
refreshHostReady()
return
// PiP releases the mic between utterances (and after drain while
// processing). Only reassert when an utterance is actively recording
// with capture already running otherwise a foreground bounce was
// cold-starting the mic mid-finalize (`!pri` / session churn).
if usesPiPKeepAlive {
guard isUtteranceRecording, capture.running else {
refreshHostReady()
return
}
}
// A system interruption (call / Siri) may be in progress. Probe it:
// `setActive(true)` inside `reassertIfRunning` fails while the
@@ -1262,29 +1268,51 @@ final class FlowSessionManager: ObservableObject {
private func handleStartRecordingCommand(utteranceId: UUID?, commandSeq: Int64) async {
if usesPiPKeepAlive {
refreshHostReady()
let micReady = await ensureCaptureReadyForPiPUtterance()
guard micReady else {
// Open the mic and utterance gate ASAP. Waiting for audio proof
// *before* beginUtterance left spin-up frames in a tiny preroll
// while the keyboard already showed "recording" users spoke into
// a closed gate and got ~1s PCM for a multi-second press.
guard startCaptureForPiPUtteranceIfNeeded() else {
failUtterance(
message: AppL10n.string("flow.coldStart.error.audioTimeout"),
kind: .audioUnavailable
)
return
}
beginUtterance(
utteranceId: utteranceId,
commandSeq: commandSeq,
requireRecentAudio: false
)
if capture.engineHasRecentAudio(maxAge: 2) {
return
}
let micReady = await capture.awaitAudioFlowing(
timeout: Self.coldStartAudioProofTimeout
)
if !micReady {
failUtterance(
message: AppL10n.string("flow.coldStart.error.audioTimeout"),
kind: .audioUnavailable
)
}
return
}
beginUtterance(utteranceId: utteranceId, commandSeq: commandSeq)
}
private func ensureCaptureReadyForPiPUtterance() async -> Bool {
/// Start capture for a PiP utterance without blocking on the first frame.
private func startCaptureForPiPUtteranceIfNeeded() -> Bool {
if capture.engineHasRecentAudio(maxAge: 2) {
return true
}
do {
try capture.start()
return true
} catch {
debug("PiP utterance capture start failed: \(error.localizedDescription)")
return false
}
return await capture.awaitAudioFlowing(timeout: Self.coldStartAudioProofTimeout)
}
private func releaseCaptureAfterPiPUtteranceIfNeeded() {
@@ -1297,14 +1325,31 @@ final class FlowSessionManager: ObservableObject {
refreshHostReady()
}
private func beginUtterance(utteranceId: UUID? = nil, commandSeq: Int64 = 0) {
guard capture.engineHasRecentAudio(maxAge: 2) else {
traceState("beginUtterance.blocked", extra: "reason=audioNotRecent")
failUtterance(
message: AppL10n.string("flow.error.audioUnavailable"),
kind: .audioUnavailable
)
return
private func beginUtterance(
utteranceId: UUID? = nil,
commandSeq: Int64 = 0,
requireRecentAudio: Bool = true
) {
if requireRecentAudio {
guard capture.engineHasRecentAudio(maxAge: 2) else {
traceState("beginUtterance.blocked", extra: "reason=audioNotRecent")
failUtterance(
message: AppL10n.string("flow.error.audioUnavailable"),
kind: .audioUnavailable
)
return
}
} else {
// PiP cold path: capture was just started; open the gate so the
// first tap frames enter the ASR stream instead of preroll only.
guard capture.running || capture.engineIsLive else {
traceState("beginUtterance.blocked", extra: "reason=captureNotRunning")
failUtterance(
message: AppL10n.string("flow.error.audioUnavailable"),
kind: .audioUnavailable
)
return
}
}
guard !isUtteranceProcessing else {
traceState("beginUtterance.ignored", extra: "reason=processing")
@@ -1339,8 +1384,18 @@ final class FlowSessionManager: ObservableObject {
let locale = SpeechLocaleResolver.resolve(localeId)
let stream = capture.beginUtterance()
let pipeline = ChunkedUtterancePipeline(asr: asr, locale: locale)
chunkedPipeline = pipeline
let useStreaming =
store.engineMode == "cloud"
&& CloudASRModelCatalog.supportsTrueStreamingASR(for: store.asrProviderId)
let pipeline: ChunkedUtterancePipeline?
if useStreaming {
chunkedPipeline = nil
pipeline = nil
} else {
let created = ChunkedUtterancePipeline(asr: asr, locale: locale)
chunkedPipeline = created
pipeline = created
}
isUtteranceRecording = true
utteranceRecordingStartedAt = Date()
@@ -1349,18 +1404,33 @@ final class FlowSessionManager: ObservableObject {
updateLiveActivityPhase(.recording)
FlowDiagnostics.log(
"beginUtterance engine=\(store.engineMode) " +
"asrType=\(type(of: asr)) pipelined=true " +
"asrType=\(type(of: asr)) streaming=\(useStreaming) " +
"localCustomLM=\(store.localASRCustomLanguageModelEnabled) " +
"max=\(Int(FlowSessionKeys.maxUtteranceDuration))s"
)
let cloudASRForStreaming = useStreaming ? (asr as? CloudASRService) : nil
asrTask = Task.detached(priority: .userInitiated) { [weak manager = self] in
let outcome = await pipeline.transcribe(stream: stream) { partial in
Task { @MainActor in
guard let manager else { return }
manager.currentPartial = partial
manager.storeCurrentPartial(partial)
let outcome: ChunkedUtterancePipelineOutcome
if let cloud = cloudASRForStreaming {
outcome = await cloud.transcribeUtteranceStreaming(stream: stream, locale: locale) { partial in
Task { @MainActor in
guard let manager else { return }
manager.currentPartial = partial
manager.storeCurrentPartial(partial)
}
}
} else if let pipeline {
outcome = await pipeline.transcribe(stream: stream) { partial in
Task { @MainActor in
guard let manager else { return }
manager.currentPartial = partial
manager.storeCurrentPartial(partial)
}
}
} else {
outcome = .failure(SharedL10n.string("error.asr.noSpeech"))
}
// Re-bind `manager` inside the `@MainActor` block so the
// weak reference is captured under the right isolation. Swift
@@ -1369,7 +1439,7 @@ final class FlowSessionManager: ObservableObject {
await MainActor.run { [weak manager] in
guard let manager else { return }
FlowDiagnostics.log(
"chunkedASR finished partialLen=\(manager.currentPartial.count) " +
"asr finished streaming=\(useStreaming) partialLen=\(manager.currentPartial.count) " +
"finalPending=\(manager.lastFinal.isEmpty)"
)
switch outcome {
@@ -1379,7 +1449,19 @@ final class FlowSessionManager: ObservableObject {
manager.currentPartial = ""
case .failure(let message):
manager.debug("asr error: \(message)")
if manager.isUtteranceRecording {
// Prefer any non-empty partial over a hard no-speech failure.
// finishProcessing used to clear bestPartialSnapshot and race
// finalize into an empty transcript even when ASR had text.
let recovery = [
manager.currentPartial,
manager.bestPartialSnapshot
]
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
.first(where: { !$0.isEmpty })
if let recovery {
manager.lastFinal = recovery
manager.debug("asr error recovered via partial len=\(recovery.count)")
} else if manager.isUtteranceRecording {
manager.failUtterance(message: message, kind: .asrFailed)
} else if manager.isUtteranceProcessing {
manager.finishProcessing(withError: message, kind: .asrFailed)