feat: add streaming cloud ASR, polish routing, and settings card layout

Unify Bailian/Volcengine/OpenAI realtime streaming, ABE polish routing with fun styles, and a shared card-page Settings hierarchy; bump to 1.1 (build 32).
2026-07-28 20:25:17 +08:00
parent 956331a2af
commit d656bac8c3
58 changed files with 4153 additions and 1396 deletions
@@ -0,0 +1,135 @@
+// CloudASRStreaming.swift
+// OSGKeyboard · Shared
+//
+// Utterance-scoped cloud ASR sessions: one long-lived connection per press,
+// streaming PCM up and interim text down. Chunked batch ASR remains the
+// fallback for providers without a true streaming protocol.
+
+import Foundation
+
+/// Long-lived cloud ASR session for one Flow utterance.
+public protocol CloudASRStreamingSession: Sendable {
+    /// Append 16 kHz mono Float32 PCM captured while the mic is open.
+    func append(samples: [Float]) async throws
+    /// Signal end-of-audio and wait for the polish-ready final transcript.
+    func finish() async throws -> String
+    func cancel()
+}
+
+/// Providers that can open an utterance-level streaming session.
+public protocol CloudASRStreamingCapable: CloudASRTranscribing {
+    func openStreamingSession(
+        locale: Locale,
+        dictionary: PersonalDictionary,
+        onPartial: @escaping @Sendable (String) -> Void
+    ) async throws -> any CloudASRStreamingSession
+}
+
+/// Feeds a live mic stream into a cloud streaming session and mirrors the
+/// existing `ChunkedUtterancePipelineOutcome` surface for Flow.
+public actor StreamingUtterancePipeline {
+    private let client: any CloudASRStreamingCapable
+    private let locale: Locale
+    private let dictionary: PersonalDictionary
+    private var cancelled = false
+    private var activeSession: (any CloudASRStreamingSession)?
+
+    public init(
+        client: any CloudASRStreamingCapable,
+        locale: Locale,
+        dictionary: PersonalDictionary
+    ) {
+        self.client = client
+        self.locale = locale
+        self.dictionary = dictionary
+    }
+
+    public func cancel() {
+        cancelled = true
+        activeSession?.cancel()
+    }
+
+    public func transcribe(
+        stream: AsyncStream<AudioBufferSnapshot>,
+        onPartial: @Sendable @escaping (String) -> Void,
+        preopenedSession: (any CloudASRStreamingSession)? = nil
+    ) async -> ChunkedUtterancePipelineOutcome {
+        cancelled = false
+        do {
+            let session: any CloudASRStreamingSession
+            if let preopenedSession {
+                session = preopenedSession
+            } else {
+                session = try await client.openStreamingSession(
+                    locale: locale,
+                    dictionary: dictionary,
+                    onPartial: onPartial
+                )
+            }
+            activeSession = session
+
+            for await snap in stream {
+                if cancelled || Task.isCancelled {
+                    session.cancel()
+                    return .cancelled
+                }
+                guard !snap.samples.isEmpty else { continue }
+                try await session.append(samples: snap.samples)
+            }
+
+            if cancelled || Task.isCancelled {
+                session.cancel()
+                return .cancelled
+            }
+
+            let finalText = try await session.finish()
+                .trimmingCharacters(in: .whitespacesAndNewlines)
+            activeSession = nil
+            guard !finalText.isEmpty else {
+                return .failure(SharedL10n.string("error.asr.noSpeech"))
+            }
+            return .success(ChunkedUtteranceSuccess(text: finalText))
+        } catch is CancellationError {
+            activeSession?.cancel()
+            activeSession = nil
+            return .cancelled
+        } catch {
+            activeSession?.cancel()
+            activeSession = nil
+            if cancelled || Task.isCancelled { return .cancelled }
+            return .failure(error.localizedDescription)
+        }
+    }
+}
+
+/// Shared PCM helpers for streaming cloud clients.
+enum CloudASRStreamingPCM {
+    static func pcm16LE(samples: [Float]) -> Data {
+        var data = Data()
+        data.reserveCapacity(samples.count * 2)
+        for sample in samples {
+            let scaled = sample * 32_767.0
+            let clipped = Swift.max(-32_768.0, Swift.min(32_767.0, scaled))
+            var littleEndian = Int16(clipped.rounded()).littleEndian
+            withUnsafeBytes(of: &littleEndian) { data.append(contentsOf: $0) }
+        }
+        return data
+    }
+
+    /// Linear upsample 16 kHz → 24 kHz for OpenAI Realtime PCM input.
+    static func upsample16kTo24k(_ samples: [Float]) -> [Float] {
+        guard !samples.isEmpty else { return [] }
+        let outCount = max(1, samples.count * 3 / 2)
+        var output = [Float]()
+        output.reserveCapacity(outCount)
+        let lastIndex = samples.count - 1
+        for i in 0..<outCount {
+            let src = Double(i) * 16.0 / 24.0
+            let i0 = min(Int(src), lastIndex)
+            let i1 = min(i0 + 1, lastIndex)
+            let frac = Float(src - Double(i0))
+            output.append(samples[i0] + (samples[i1] - samples[i0]) * frac)
+        }
+        return output
+    }
+}