feat: add streaming cloud ASR, polish routing, and settings card layout

Unify Bailian/Volcengine/OpenAI realtime streaming, ABE polish routing with
fun styles, and a shared card-page Settings hierarchy; bump to 1.1 (build 32).
This commit is contained in:
Rocky
2026-07-28 20:25:17 +08:00
parent 956331a2af
commit d656bac8c3
58 changed files with 4153 additions and 1396 deletions
@@ -1,8 +1,9 @@
// CloudASRService.swift
// OSGKeyboard · Shared
//
// Cloud-engine ASR: uploads PCM chunks to the user's configured provider
// with personal-dictionary bias. Moonshot falls back to on-device ASR.
// Cloud-engine ASR: uploads PCM to the user's configured provider with
// personal-dictionary bias. Streaming-capable providers use one utterance
// WebSocket; others stay on chunked batch. Moonshot falls back to on-device ASR.
import Foundation
import os
@@ -16,6 +17,7 @@ public final class CloudASRService: ASRService, @unchecked Sendable {
private var usesLocalFallback = false
private var boundProviderId: String?
private var cancelled = false
private var streamingPipeline: StreamingUtterancePipeline?
public init(
store: any ConfigurationStore = AppGroupStore(),
@@ -29,6 +31,11 @@ public final class CloudASRService: ASRService, @unchecked Sendable {
self.localFallback = localFallback ?? SpeechAnalyzerASR()
}
/// Whether Flow should prefer utterance-level true streaming for the bound provider.
public var supportsUtteranceStreaming: Bool {
CloudASRModelCatalog.supportsTrueStreamingASR(for: store.asrProviderId)
}
public func resetForNewUtterance() {
lock.withLock { cancelled = false }
if usesLocalFallback {
@@ -79,6 +86,55 @@ public final class CloudASRService: ASRService, @unchecked Sendable {
}
}
/// Utterance-level streaming; if the session cannot start, fall back to
/// chunked batch on the same mic stream. Mid-stream failures surface as
/// errors (finalize still has PCM batch fallback).
public func transcribeUtteranceStreaming(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale,
onPartial: @escaping @Sendable (String) -> Void
) async -> ChunkedUtterancePipelineOutcome {
bindClientIfNeeded()
if usesLocalFallback {
let pipeline = ChunkedUtterancePipeline(asr: localFallback, locale: locale)
return await pipeline.transcribe(stream: stream, onPartial: onPartial)
}
guard let streamingClient = lock.withLock({ client as? CloudASRStreamingCapable }) else {
let pipeline = ChunkedUtterancePipeline(asr: self, locale: locale)
return await pipeline.transcribe(stream: stream, onPartial: onPartial)
}
let session: any CloudASRStreamingSession
do {
session = try await streamingClient.openStreamingSession(
locale: locale,
dictionary: store.personalDictionary,
onPartial: onPartial
)
} catch {
OSGLog.asr.warning(
"streaming ASR session open failed, using chunked batch: \(error.localizedDescription, privacy: .public)"
)
let pipeline = ChunkedUtterancePipeline(asr: self, locale: locale)
return await pipeline.transcribe(stream: stream, onPartial: onPartial)
}
let pipeline = StreamingUtterancePipeline(
client: streamingClient,
locale: locale,
dictionary: store.personalDictionary
)
lock.withLock { streamingPipeline = pipeline }
let outcome = await pipeline.transcribe(
stream: stream,
onPartial: onPartial,
preopenedSession: session
)
lock.withLock { streamingPipeline = nil }
return outcome
}
public func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
@@ -88,6 +144,34 @@ public final class CloudASRService: ASRService, @unchecked Sendable {
return localFallback.transcribe(stream: stream, locale: locale)
}
if supportsUtteranceStreaming, lock.withLock({ client is CloudASRStreamingCapable }) {
return AsyncStream { continuation in
continuation.yield(.capability(onDeviceSupported: false))
let task = Task {
let outcome = await self.transcribeUtteranceStreaming(
stream: stream,
locale: locale,
onPartial: { partial in
continuation.yield(.partial(partial))
}
)
switch outcome {
case .success(let success):
continuation.yield(.final(success.text))
case .failure(let message):
continuation.yield(.error(message))
case .cancelled:
break
}
continuation.finish()
}
continuation.onTermination = { @Sendable _ in
task.cancel()
self.cancel()
}
}
}
return AsyncStream { continuation in
continuation.yield(.capability(onDeviceSupported: false))
let task = Task {
@@ -129,6 +213,8 @@ public final class CloudASRService: ASRService, @unchecked Sendable {
public func cancel() {
lock.withLock { cancelled = true }
let pipeline = lock.withLock { streamingPipeline }
Task { await pipeline?.cancel() }
localFallback.cancel()
}