feat(polish): allow mood emoji on custom styles and ship Flow/ASR fixes

Custom polish styles can opt in to emotion-matched emoji (default off), with prompt-level opt-in detection so paste-only styles keep model-added emoji. Also include Volcengine API-Key ASR auth, voice-processing capture, PiP flash fix, and related keyboard Shift/haptics reliability work.
2026-08-06 15:11:12 +08:00
parent 2ed16e1fbf
commit a8d58d8f0c
47 changed files with 1467 additions and 258 deletions
@@ -29,19 +29,15 @@ struct VolcengineCloudASRClient: CloudASRTranscribing, CloudASRStreamingCapable
        onPartial: @escaping @Sendable (String) -> Void
    ) async throws -> any CloudASRStreamingSession {
        _ = locale
-        let credentials = try VolcengineCredentials.parse(
-            apiKey: apiKey,
-            fallbackResourceID: resolvedResourceID
-        )
+        _ = resourceID // Product is locked to Doubao streaming 2.0 duration.
+        let credentials = VolcengineASRFields.parse(apiKey: apiKey)
+        guard credentials.hasUsableCredentials else { throw CloudASRError.noAPIKey }
        let url = try resolvedEndpointURL()
        let connectID = UUID().uuidString

        var request = URLRequest(url: url)
        request.timeoutInterval = 8
-        request.setValue(credentials.appID, forHTTPHeaderField: "X-Api-App-Key")
-        request.setValue(credentials.accessToken, forHTTPHeaderField: "X-Api-Access-Key")
-        request.setValue(credentials.resourceID, forHTTPHeaderField: "X-Api-Resource-Id")
-        request.setValue(connectID, forHTTPHeaderField: "X-Api-Connect-Id")
+        credentials.applyWebSocketAuthHeaders(to: &request, connectID: connectID)

        let task = session.webSocketTask(with: request)
        task.resume()
@@ -90,12 +86,6 @@ struct VolcengineCloudASRClient: CloudASRTranscribing, CloudASRStreamingCapable
        live.cancel()
    }

-    private var resolvedResourceID: String {
-        resourceID.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
-            ? CloudASRModelCatalog.volcengineDefaultResourceID
-            : resourceID.trimmingCharacters(in: .whitespacesAndNewlines)
-    }
-
    private func resolvedEndpointURL() throws -> URL {
        let raw = endpoint.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
            ? CloudASRModelCatalog.volcengineEndpoint
@@ -424,46 +414,6 @@ private final class VolcengineStreamingSession: CloudASRStreamingSession, @unche
    }
 }

-private struct VolcengineCredentials {
-    let appID: String
-    let accessToken: String
-    let resourceID: String
-
-    static func parse(apiKey: String, fallbackResourceID: String) throws -> VolcengineCredentials {
-        let trimmed = apiKey.trimmingCharacters(in: .whitespacesAndNewlines)
-        guard !trimmed.isEmpty else { throw CloudASRError.noAPIKey }
-
-        if let data = trimmed.data(using: .utf8),
-           let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {
-            let appID = string(json, keys: ["app_id", "appId", "appid"])
-            let token = string(json, keys: ["access_token", "accessToken", "token"])
-            let resourceID = string(json, keys: ["resource_id", "resourceId", "resource"])
-                ?? fallbackResourceID
-            guard let appID, let token, !resourceID.isEmpty else { throw CloudASRError.noAPIKey }
-            return VolcengineCredentials(appID: appID, accessToken: token, resourceID: resourceID)
-        }
-
-        let separators = CharacterSet(charactersIn: ":\n,")
-        let parts = trimmed
-            .components(separatedBy: separators)
-            .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
-            .filter { !$0.isEmpty }
-        guard parts.count >= 2 else { throw CloudASRError.noAPIKey }
-        let resourceID = parts.count >= 3 ? parts[2] : fallbackResourceID
-        return VolcengineCredentials(appID: parts[0], accessToken: parts[1], resourceID: resourceID)
-    }
-
-    private static func string(_ json: [String: Any], keys: [String]) -> String? {
-        for key in keys {
-            if let value = json[key] as? String {
-                let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
-                if !trimmed.isEmpty { return trimmed }
-            }
-        }
-        return nil
-    }
-}
-
 enum VolcengineMessageType: UInt8 {
    case fullClientRequest = 0b0001
    case audioOnlyRequest = 0b0010
@@ -68,10 +68,13 @@ public final class FlowAudioSessionCoordinator: @unchecked Sendable {
    public func activateCapture() async throws -> FlowAudioSessionSnapshot {
        let activation: CaptureActivation = try await perform {
            if self.mode != .capture {
+                // `.voiceChat` turns on system speech DSP (noise suppression /
+                // AGC). `.measurement` feeds near-raw PCM and is weak against
+                // a competing talker in the same room.
                try self.session.setCategory(
                    .playAndRecord,
-                    mode: .measurement,
-                    options: [.defaultToSpeaker, .allowBluetoothHFP, .mixWithOthers]
+                    mode: FlowCaptureVoiceProcessing.captureMode,
+                    options: FlowCaptureVoiceProcessing.captureOptions
                )
            }
            if !self.active {
@@ -95,6 +98,9 @@ public final class FlowAudioSessionCoordinator: @unchecked Sendable {
                        category: "flow"
                    )
                }
+            } else {
+                // Built-in near-talk preference (cardioid when available).
+                FlowCaptureVoiceProcessing.preferNearTalkBuiltInMic(on: self.session)
            }
            self.mode = .capture
            return CaptureActivation(
@@ -0,0 +1,108 @@
+// FlowCaptureVoiceProcessing.swift
+// OSGKeyboard · Host Support
+//
+// Capture-side speech front-end for local ASR: Apple Voice Processing
+// (noise suppression / AGC / AEC) plus near-talk built-in mic preference.
+// `.measurement` delivers near-raw PCM and is a poor fit for competing
+// talkers in the same room; `.voiceChat` + VP is the system path that
+// also unlocks Control Center Mic Modes (incl. Voice Isolation).
+
+import AVFoundation
+import Foundation
+import OSGKeyboardShared
+
+public enum FlowCaptureVoiceProcessing {
+
+    /// Speech-oriented session mode. Prefer over `.measurement` for ASR.
+    public static let captureMode: AVAudioSession.Mode = .voiceChat
+
+    public static let captureOptions: AVAudioSession.CategoryOptions = [
+        .defaultToSpeaker, .allowBluetoothHFP, .mixWithOthers
+    ]
+
+    /// Enable Apple Voice Processing on a stopped engine. Format may change
+    /// afterward — callers must re-read `inputNode` formats before `installTap`.
+    @discardableResult
+    public static func enableVoiceProcessing(on engine: AVAudioEngine) -> Bool {
+        let input = engine.inputNode
+        if input.isVoiceProcessingEnabled { return true }
+        do {
+            try input.setVoiceProcessingEnabled(true)
+            OSGDiag.log("voiceProcessing enabled", category: "flow")
+            logActiveMicrophoneMode()
+            return true
+        } catch {
+            OSGDiag.log(
+                "voiceProcessing enable failed: \(error.localizedDescription)",
+                category: "flow"
+            )
+            return false
+        }
+    }
+
+    /// Prefer a near-field built-in data source (front/lower + cardioid when
+    /// available). No-op while Bluetooth HFP is preferred or active — keep
+    /// the existing headset route.
+    public static func preferNearTalkBuiltInMic(on session: AVAudioSession) {
+        if session.currentRoute.inputs.first?.portType == .bluetoothHFP { return }
+        if session.preferredInput?.portType == .bluetoothHFP { return }
+
+        guard let builtIn = session.availableInputs?.first(where: {
+            $0.portType == .builtInMic
+        }) else { return }
+
+        let sources = builtIn.dataSources ?? []
+        let preferred =
+            sources.first(where: { $0.orientation == .front })
+            ?? sources.first(where: { $0.location == .lower })
+            ?? sources.first
+
+        if let preferred {
+            if preferred.supportedPolarPatterns?.contains(.cardioid) == true {
+                do {
+                    try preferred.setPreferredPolarPattern(.cardioid)
+                } catch {
+                    OSGDiag.log(
+                        "cardioid polar pattern failed: \(error.localizedDescription)",
+                        category: "flow"
+                    )
+                }
+            }
+            do {
+                try builtIn.setPreferredDataSource(preferred)
+            } catch {
+                OSGDiag.log(
+                    "preferred data source failed: \(error.localizedDescription)",
+                    category: "flow"
+                )
+            }
+        }
+
+        do {
+            try session.setPreferredInput(builtIn)
+        } catch {
+            OSGDiag.log(
+                "preferred built-in mic failed: \(error.localizedDescription)",
+                category: "flow"
+            )
+        }
+    }
+
+    public static func logActiveMicrophoneMode() {
+        let preferred = micModeLabel(AVCaptureDevice.preferredMicrophoneMode)
+        let active = micModeLabel(AVCaptureDevice.activeMicrophoneMode)
+        OSGDiag.log(
+            "micMode preferred=\(preferred) active=\(active)",
+            category: "flow"
+        )
+    }
+
+    private static func micModeLabel(_ mode: AVCaptureDevice.MicrophoneMode) -> String {
+        switch mode {
+        case .standard: return "standard"
+        case .wideSpectrum: return "wideSpectrum"
+        case .voiceIsolation: return "voiceIsolation"
+        @unknown default: return "unknown(\(mode.rawValue))"
+        }
+    }
+}
@@ -604,13 +604,25 @@ public final class FlowContinuousCapture {
            FlowAudioEngineHandle(audioEngine)
        )

+        // Voice Processing must be enabled while the engine is stopped, and
+        // it can change the RemoteIO format — enable first, then sync rates.
        var candidateEngine = AVAudioEngine()
+        var voiceProcessingOn = FlowCaptureVoiceProcessing.enableVoiceProcessing(
+            on: candidateEngine
+        )
        var inputNode = candidateEngine.inputNode
        var hardwareFormat = inputNode.inputFormat(forBus: 0)
        var outputFormat = inputNode.outputFormat(forBus: 0)
-        for _ in 0..<3 where abs(hardwareFormat.sampleRate - sessionSnapshot.sampleRate) >= 1 {
+        var resolvedSession = sessionSnapshot
+        for _ in 0..<3 where abs(hardwareFormat.sampleRate - resolvedSession.sampleRate) >= 1 {
            try? await Task.sleep(nanoseconds: 50_000_000)
+            // Re-enter capture so the session rate tracks VP / route shifts.
+            resolvedSession = (try? await FlowAudioSessionCoordinator.shared.activateCapture())
+                ?? resolvedSession
            candidateEngine = AVAudioEngine()
+            voiceProcessingOn = FlowCaptureVoiceProcessing.enableVoiceProcessing(
+                on: candidateEngine
+            )
            inputNode = candidateEngine.inputNode
            hardwareFormat = inputNode.inputFormat(forBus: 0)
            outputFormat = inputNode.outputFormat(forBus: 0)
@@ -619,8 +631,9 @@ public final class FlowContinuousCapture {
            "audioSession.active",
            "hwRate=\(Int(hardwareFormat.sampleRate)) hwChannels=\(hardwareFormat.channelCount) "
                + "outputRate=\(Int(outputFormat.sampleRate)) "
-                + "sessionRate=\(Int(sessionSnapshot.sampleRate)) "
-                + "route=\(sessionSnapshot.inputPortType)"
+                + "sessionRate=\(Int(resolvedSession.sampleRate)) "
+                + "route=\(resolvedSession.inputPortType) "
+                + "voiceProcessing=\(voiceProcessingOn ? 1 : 0)"
        )
        guard hardwareFormat.sampleRate > 0, hardwareFormat.channelCount > 0 else {
            FlowTrace.warn(
@@ -632,8 +645,8 @@ public final class FlowContinuousCapture {
                channels: Int(hardwareFormat.channelCount)
            )
        }
-        guard sessionSnapshot.sampleRate <= 0
-                || abs(hardwareFormat.sampleRate - sessionSnapshot.sampleRate) < 1 else {
+        guard resolvedSession.sampleRate <= 0
+                || abs(hardwareFormat.sampleRate - resolvedSession.sampleRate) < 1 else {
            throw StartError.invalidHardwareFormat(
                sampleRate: hardwareFormat.sampleRate,
                channels: Int(hardwareFormat.channelCount)
@@ -703,7 +716,7 @@ public final class FlowContinuousCapture {
            throw StartError.engineStartFailed(error.localizedDescription)
        }
        audioEngine = candidateEngine
-        activeRouteSnapshot = sessionSnapshot
+        activeRouteSnapshot = resolvedSession
        engineActivationCount += 1
        lastActivationAt = Date()
        FlowTrace.capture("engine.started", "running=\(audioEngine.isRunning ? 1 : 0)")
@@ -177,30 +177,27 @@ public final class LiveDictationController: ObservableObject {

        // 3. Audio session — only configure once per process.
        //
-        // Category is `.record` (not `.playAndRecord`) because the
-        // preview never plays back audio — it just records from the
-        // mic and hands the buffers to `SpeechAnalyzer`. On the
-        // iOS Simulator, `.playAndRecord` requires the
-        // `AURemoteIO` Audio Unit's *output* side to also be
-        // enabled, but the simulator's "speaker" reports a 0 Hz
-        // hardware format, so `AURemoteIO::enable` fails with
-        // `kAudioUnitErr_FormatNotSupported` (-10851) and any
-        // subsequent `installTap` traps with "Failed to create tap
-        // due to format mismatch". `.record` skips the output
-        // side entirely, so the simulator can record.
-        //
-        // The real keyboard extension (`OSGKeyboardExt`) keeps
-        // `.playAndRecord` because it runs on a real device where
-        // the output side has a real hardware format, and may want
-        // to play click sounds / haptic feedback. Only the preview
-        // needs the simulator-friendly category.
+        // On device we use `.playAndRecord` + `.voiceChat` so Apple Voice
+        // Processing can suppress competing talkers. The iOS Simulator's
+        // speaker reports a 0 Hz output format, so `.playAndRecord` fails
+        // with `kAudioUnitErr_FormatNotSupported` (-10851); keep `.record`
+        // there so preview still works under CoreSimulator.
        if !didConfigureAudioSession {
            do {
                let session = AVAudioSession.sharedInstance()
-                try session.setCategory(.record,
-                                        mode: .measurement,
-                                        options: [])
+                #if targetEnvironment(simulator)
+                try session.setCategory(.record, mode: .default, options: [])
+                #else
+                try session.setCategory(
+                    .playAndRecord,
+                    mode: FlowCaptureVoiceProcessing.captureMode,
+                    options: FlowCaptureVoiceProcessing.captureOptions
+                )
+                #endif
                try session.setActive(true, options: .notifyOthersOnDeactivation)
+                #if !targetEnvironment(simulator)
+                FlowCaptureVoiceProcessing.preferNearTalkBuiltInMic(on: session)
+                #endif
                didConfigureAudioSession = true
            } catch {
                debug("audio session failed: \(error.localizedDescription)")
@@ -214,6 +211,8 @@ public final class LiveDictationController: ObservableObject {
        // The route may have changed while the preview was closed. A fresh
        // engine created after session activation avoids a stale input node.
        audioEngine = AVAudioEngine()
+        // Enable VP before reading hardware format — RemoteIO rate can shift.
+        _ = FlowCaptureVoiceProcessing.enableVoiceProcessing(on: audioEngine)

        // 4. Spin up the engine + ASR.
        phase = .recording