// VolcengineCloudASRClient.swift // OSGKeyboard · HostSupport // // Volcengine SAUC bigmodel ASR client. Utterance-level WebSocket session with // enable_nonstream (official two-pass): interim text for on-screen partials, // definite utterances for polish-ready finals. import Foundation import os #if canImport(OSGKeyboardShared) import OSGKeyboardShared #endif struct VolcengineCloudASRClient: CloudASRTranscribing, CloudASRStreamingCapable { let apiKey: String let endpoint: String let resourceID: String let session: URLSession static let targetChunkBytes = 6_400 // 200 ms @ 16 kHz, 16-bit, mono. static let finalTimeout: TimeInterval = 12 private static let hotwordCap = 80 func prepare(dictionary: PersonalDictionary) async throws {} func openStreamingSession( locale: Locale, dictionary: PersonalDictionary, onPartial: @escaping @Sendable (String) -> Void ) async throws -> any CloudASRStreamingSession { _ = locale _ = resourceID // Product is locked to Doubao streaming 2.0 duration. let credentials = VolcengineASRFields.parse(apiKey: apiKey) guard credentials.hasUsableCredentials else { throw CloudASRError.noAPIKey } let url = try resolvedEndpointURL() let connectID = UUID().uuidString var request = URLRequest(url: url) request.timeoutInterval = 8 credentials.applyWebSocketAuthHeaders(to: &request, connectID: connectID) let task = session.webSocketTask(with: request) task.resume() let live = VolcengineStreamingSession( wsTask: task, connectID: connectID, dictionary: dictionary, onPartial: onPartial ) try await live.start() return live } func transcribe( samples: [Float], sampleRate: Int, locale: Locale, dictionary: PersonalDictionary ) async throws -> String { _ = sampleRate guard !samples.isEmpty else { throw CloudASRError.emptyTranscript } let session = try await openStreamingSession( locale: locale, dictionary: dictionary, onPartial: { _ in } ) try await session.append(samples: samples) let text = try await session.finish() let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { throw CloudASRError.emptyTranscript } return trimmed } /// Settings connection probe: WebSocket upgrade + first-frame auth only. /// /// Do not push silence through `transcribe` — SAUC returns an empty final /// then closes the socket, and `finish()` prefers that Socket error over /// `emptyTranscript`, so the default silence probe always failed while /// real dictation (with speech) still worked. func probeConnection() async throws { let live = try await openStreamingSession( locale: Locale(identifier: "zh-CN"), dictionary: .empty, onPartial: { _ in } ) live.cancel() } private func resolvedEndpointURL() throws -> URL { let raw = endpoint.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? CloudASRModelCatalog.volcengineEndpoint : endpoint.trimmingCharacters(in: .whitespacesAndNewlines) guard let url = URL(string: raw) else { throw CloudASRError.invalidURL } return url } static func firstFramePayload( connectID: String, dictionary: PersonalDictionary ) throws -> Data { var request: [String: Any] = [ "model_name": "bigmodel", "enable_itn": true, "enable_punc": true, "show_utterances": true, "enable_speaker_info": true, // Official two-pass: stream interim for UI, nostream re-decode per // VAD sentence for definite polish-ready text (scheme A). "enable_nonstream": true, "end_window_size": 800, "force_to_speech_time": 1_000 ] if let context = hotwordContext(dictionary: dictionary) { request["context"] = context } let payload: [String: Any] = [ "user": ["uid": connectID], "audio": [ "format": "pcm", "rate": 16_000, "bits": 16, "channel": 1, "codec": "raw" ], "request": request ] return try JSONSerialization.data(withJSONObject: payload) } private static func hotwordContext(dictionary: PersonalDictionary) -> String? { var seen: [String] = [] for word in dictionary.asrHotwords() { let trimmed = word.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { continue } guard !seen.contains(where: { $0.caseInsensitiveCompare(trimmed) == .orderedSame }) else { continue } seen.append(trimmed) if seen.count >= hotwordCap { break } } guard !seen.isEmpty else { return nil } let words = seen.map { ["word": $0] } guard let data = try? JSONSerialization.data(withJSONObject: ["hotwords": words]) else { return nil } return String(data: data, encoding: .utf8) } static func displayText(from payload: Data) -> String { guard let json = try? JSONSerialization.jsonObject(with: payload) as? [String: Any], let result = normalizedResult(from: json) else { return "" } if let utterances = result["utterances"] as? [[String: Any]], !utterances.isEmpty { let pieces = utterances.compactMap { $0["text"] as? String } let joined = pieces.joined() if !joined.isEmpty { return joined } } return result["text"] as? String ?? "" } /// Prefer definite (two-pass) utterance text for polish input. static func committedText(from payload: Data) -> String { guard let json = try? JSONSerialization.jsonObject(with: payload) as? [String: Any], let result = normalizedResult(from: json), let utterances = result["utterances"] as? [[String: Any]], !utterances.isEmpty else { return "" } let definite = utterances.compactMap { utterance -> String? in let isDefinite = utterance["definite"] as? Bool ?? false guard isDefinite else { return nil } return utterance["text"] as? String } return definite.joined() } private static func normalizedResult(from json: [String: Any]) -> [String: Any]? { if let result = json["result"] as? [String: Any] { return result } if let results = json["result"] as? [[String: Any]] { return results.first } if json["text"] is String { return json } return nil } } // MARK: - Utterance session private final class VolcengineStreamingSession: CloudASRStreamingSession, @unchecked Sendable { private let wsTask: URLSessionWebSocketTask private let connectID: String private let dictionary: PersonalDictionary private let onPartial: @Sendable (String) -> Void private let lock = OSAllocatedUnfairLock() private var sequence: Int32 = 1 private var pcmBuffer = Data() private var receiveTask: Task? private var failure: Error? private var finished = false private var lastDisplay = "" private var lastCommitted = "" private var sawServerFinal = false init( wsTask: URLSessionWebSocketTask, connectID: String, dictionary: PersonalDictionary, onPartial: @escaping @Sendable (String) -> Void ) { self.wsTask = wsTask self.connectID = connectID self.dictionary = dictionary self.onPartial = onPartial } func start() async throws { let firstPayload = try VolcengineCloudASRClient.firstFramePayload( connectID: connectID, dictionary: dictionary ) try await send( VolcengineFrame.build( messageType: .fullClientRequest, flags: .positiveSequence, serialization: .json, payload: firstPayload, sequence: 1 ) ) sequence = 2 receiveTask = Task { [weak self] in await self?.receiveLoop() } } func append(samples: [Float]) async throws { try throwIfFailed() let pcm = CloudASRStreamingPCM.pcm16LE(samples: samples) let (frames, nextSequences): ([Data], [Int32]) = lock.withLock { pcmBuffer.append(pcm) var frames: [Data] = [] while pcmBuffer.count >= VolcengineCloudASRClient.targetChunkBytes { let frame = pcmBuffer.prefix(VolcengineCloudASRClient.targetChunkBytes) frames.append(Data(frame)) pcmBuffer.removeFirst(VolcengineCloudASRClient.targetChunkBytes) } let nextSequences: [Int32] = frames.indices.map { _ in let seq = sequence sequence += 1 return seq } return (frames, nextSequences) } for (frame, seq) in zip(frames, nextSequences) { try await send( VolcengineFrame.build( messageType: .audioOnlyRequest, flags: .positiveSequence, serialization: .none, payload: frame, sequence: seq ) ) } } func finish() async throws -> String { try throwIfFailed() // Only consume a sequence number when we actually send trailing PCM. // Skipping an unused seq (common when length is an exact chunk multiple, // e.g. the settings probe's 1 s / 32_000-byte clip) makes the final // negative packet mismatch server autoAssignedSequence → error 45000000. let trailing = lock.withLock { () -> Data in let data = pcmBuffer pcmBuffer.removeAll(keepingCapacity: false) return data } if !trailing.isEmpty { let endSequence = lock.withLock { () -> Int32 in let seq = sequence sequence += 1 return seq } try await send( VolcengineFrame.build( messageType: .audioOnlyRequest, flags: .positiveSequence, serialization: .none, payload: trailing, sequence: endSequence ) ) } let negativeSeq = lock.withLock { () -> Int32 in let seq = sequence sequence += 1 return seq } try await send( VolcengineFrame.build( messageType: .audioOnlyRequest, flags: .negativeSequence, serialization: .none, payload: Data(), sequence: -negativeSeq ) ) let deadline = Date().addingTimeInterval(VolcengineCloudASRClient.finalTimeout) while Date() < deadline { try throwIfFailed() let snapshot = lock.withLock { (sawServerFinal, lastCommitted, lastDisplay) } if snapshot.0 { let text = snapshot.1.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? snapshot.2 : snapshot.1 let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) cancel() if trimmed.isEmpty { throw CloudASRError.emptyTranscript } return trimmed } try await Task.sleep(nanoseconds: 20_000_000) } cancel() throw CloudASRError.transport("Volcengine final result timed out") } func cancel() { receiveTask?.cancel() wsTask.cancel(with: .normalClosure, reason: nil) lock.withLock { finished = true } } private func receiveLoop() async { while !Task.isCancelled { let message: URLSessionWebSocketTask.Message do { message = try await wsTask.receive() } catch { publishFailure(CloudASRError.transport(error.localizedDescription)) return } let data: Data switch message { case .data(let payload): data = payload case .string(let string): data = Data(string.utf8) @unknown default: continue } guard let frame = VolcengineFrame.parse(data) else { continue } if frame.messageType == .errorMessage { let code = frame.errorCode ?? 0 publishFailure( CloudASRError.transport( "ASR error code=\(code) responseBytes=\(frame.payload.count)" ) ) return } guard frame.messageType == .fullServerResponse else { continue } let display = VolcengineCloudASRClient.displayText(from: frame.payload) .trimmingCharacters(in: .whitespacesAndNewlines) let committed = VolcengineCloudASRClient.committedText(from: frame.payload) .trimmingCharacters(in: .whitespacesAndNewlines) let emit = lock.withLock { () -> String in if !display.isEmpty { lastDisplay = display } if !committed.isEmpty { lastCommitted = committed } if frame.isFinal { sawServerFinal = true } return lastDisplay } if !emit.isEmpty { onPartial(emit) } } } private func send(_ data: Data) async throws { do { try await wsTask.send(.data(data)) } catch where ProviderToolCancellation.matches(error) { throw CancellationError() } catch { throw CloudASRError.transport(error.localizedDescription) } } private func throwIfFailed() throws { let (error, done) = lock.withLock { (failure, finished) } if let error { throw error } if done { throw CloudASRError.transport("Volcengine session cancelled") } } private func publishFailure(_ error: Error) { lock.withLock { failure = error } cancel() } } enum VolcengineMessageType: UInt8 { case fullClientRequest = 0b0001 case audioOnlyRequest = 0b0010 case fullServerResponse = 0b1001 case errorMessage = 0b1111 } enum VolcengineFlags: UInt8 { case none = 0b0000 case positiveSequence = 0b0001 case lastPacket = 0b0010 case negativeSequence = 0b0011 } enum VolcengineSerialization: UInt8 { case none = 0b0000 case json = 0b0001 } struct VolcengineFrame { let messageType: VolcengineMessageType? let flags: UInt8 let sequence: Int32? let errorCode: UInt32? let payload: Data var isFinal: Bool { flags == VolcengineFlags.lastPacket.rawValue || flags == VolcengineFlags.negativeSequence.rawValue || (sequence ?? 0) < 0 } static func build( messageType: VolcengineMessageType, flags: VolcengineFlags, serialization: VolcengineSerialization, payload: Data, sequence: Int32? ) -> Data { var data = Data() data.append(0x11) data.append((messageType.rawValue << 4) | flags.rawValue) data.append(serialization.rawValue << 4) data.append(0x00) if flags == .positiveSequence || flags == .negativeSequence, let sequence { data.appendBE32(UInt32(bitPattern: sequence)) } data.appendBE32(UInt32(payload.count)) data.append(payload) return data } static func parse(_ data: Data) -> VolcengineFrame? { guard data.count >= 8 else { return nil } let bytes = [UInt8](data) let headerSize = Int(bytes[0] & 0x0F) * 4 guard headerSize >= 4, data.count >= headerSize + 4 else { return nil } let typeRaw = (bytes[1] >> 4) & 0x0F let messageType = VolcengineMessageType(rawValue: typeRaw) let flags = bytes[1] & 0x0F let compression = bytes[2] & 0x0F guard compression == 0 else { return nil } var offset = headerSize var sequence: Int32? if flags == VolcengineFlags.positiveSequence.rawValue || flags == VolcengineFlags.negativeSequence.rawValue { guard let value = data.readBE32(at: offset) else { return nil } sequence = Int32(bitPattern: value) offset += 4 } if messageType == .errorMessage { guard let code = data.readBE32(at: offset), let size = data.readBE32(at: offset + 4) else { return nil } offset += 8 guard data.count >= offset + Int(size) else { return nil } return VolcengineFrame( messageType: messageType, flags: flags, sequence: sequence, errorCode: code, payload: data.subdata(in: offset..<(offset + Int(size))) ) } guard let size = data.readBE32(at: offset) else { return nil } offset += 4 guard data.count >= offset + Int(size) else { return nil } return VolcengineFrame( messageType: messageType, flags: flags, sequence: sequence, errorCode: nil, payload: data.subdata(in: offset..<(offset + Int(size))) ) } } private extension Data { mutating func appendBE32(_ value: UInt32) { var bigEndian = value.bigEndian Swift.withUnsafeBytes(of: &bigEndian) { append(contentsOf: $0) } } func readBE32(at offset: Int) -> UInt32? { guard count >= offset + 4 else { return nil } return self[offset..<(offset + 4)].reduce(UInt32(0)) { ($0 << 8) | UInt32($1) } } }