feat: macOS architecture, cloud ASR/LLM providers, and 6-step iOS onboarding
- Add macOS menu-bar dictation app with local ASR models (SenseVoice/Qwen3),
global Option hotkey, and bottom overlay
- Add cloud ASR/LLM providers (Anthropic, Volcengine, Bailian, and more) with
provider logos, model listing, and connection checks
- Add shared 7-day usage stats UI (UsageStatsCluster / SevenDayUsageChart)
- Add iOS onboarding step 6 for polish LLM setup; hide custom-language-model
diagnostic toggle behind DEBUG
- Unify iOS onboarding tagline with the macOS brand line ("开口即文字。")
- Rewrite README (Chinese-first, product-oriented) and refresh GitHub Pages
This commit is contained in:
@@ -0,0 +1,434 @@
|
||||
// BailianRealtimeASRClient.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Alibaba Cloud Bailian / DashScope realtime ASR over the classic inference
|
||||
// WebSocket (`/api-ws/v1/inference`). Matches OpenLess' `bailian.rs` wire
|
||||
// protocol: run-task → PCM binary frames → finish-task → result events.
|
||||
|
||||
import Foundation
|
||||
|
||||
struct BailianRealtimeASRClient: CloudASRTranscribing {
|
||||
let apiKey: String
|
||||
let endpoint: String
|
||||
let model: String
|
||||
let vocabularyID: String?
|
||||
let session: URLSession
|
||||
|
||||
/// 100 ms of 16 kHz / 16-bit / mono PCM.
|
||||
private static let targetChunkBytes = 3_200
|
||||
private static let startTimeout: TimeInterval = 8
|
||||
private static let finalTimeout: TimeInterval = 12
|
||||
private static let sessionTimeout: TimeInterval = startTimeout + finalTimeout + 4
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
guard sampleRate == 16_000 else {
|
||||
throw CloudASRError.transport("Bailian realtime expects 16 kHz audio")
|
||||
}
|
||||
guard !samples.isEmpty else { throw CloudASRError.emptyTranscript }
|
||||
|
||||
let url = try resolvedEndpointURL()
|
||||
let pcm = Self.pcm16Data(samples: samples)
|
||||
let taskID = UUID().uuidString.replacingOccurrences(of: "-", with: "")
|
||||
let resolvedModel = model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? CloudASRModelCatalog.alibabaFunASRRealtime
|
||||
: model.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.timeoutInterval = 8
|
||||
request.setValue(
|
||||
"bearer \(apiKey.trimmingCharacters(in: .whitespacesAndNewlines))",
|
||||
forHTTPHeaderField: "Authorization"
|
||||
)
|
||||
|
||||
let wsTask = session.webSocketTask(with: request)
|
||||
wsTask.resume()
|
||||
|
||||
return try await withThrowingTaskGroup(of: String.self) { group in
|
||||
let events = BailianEventStream(task: wsTask)
|
||||
|
||||
group.addTask {
|
||||
defer { events.cancel() }
|
||||
return try await Self.runSession(
|
||||
taskID: taskID,
|
||||
model: resolvedModel,
|
||||
pcm: pcm,
|
||||
wsTask: wsTask,
|
||||
events: events
|
||||
)
|
||||
}
|
||||
|
||||
group.addTask {
|
||||
try await Task.sleep(nanoseconds: UInt64(Self.sessionTimeout * 1_000_000_000))
|
||||
events.cancel()
|
||||
wsTask.cancel(with: .goingAway, reason: nil)
|
||||
throw CloudASRError.transport("session timed out")
|
||||
}
|
||||
|
||||
guard let result = try await group.next() else {
|
||||
throw CloudASRError.emptyTranscript
|
||||
}
|
||||
group.cancelAll()
|
||||
return result.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
}
|
||||
|
||||
/// Settings connection probe: handshake to `task-started` only.
|
||||
///
|
||||
/// Reaching `task-started` proves endpoint + `Authorization` + model are
|
||||
/// all valid — which is exactly what "validate connection" must check.
|
||||
/// It deliberately sends NO audio: DashScope realtime rejects a short
|
||||
/// silent probe with a `task-failed: emptyAudio`, which is a false
|
||||
/// negative for a connectivity test. A real auth/quota/model failure
|
||||
/// still arrives as `task-failed` before `task-started` and surfaces.
|
||||
func probeConnection() async throws {
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
|
||||
let url = try resolvedEndpointURL()
|
||||
let taskID = UUID().uuidString.replacingOccurrences(of: "-", with: "")
|
||||
let resolvedModel = model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? CloudASRModelCatalog.alibabaFunASRRealtime
|
||||
: model.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.timeoutInterval = 8
|
||||
request.setValue(
|
||||
"bearer \(apiKey.trimmingCharacters(in: .whitespacesAndNewlines))",
|
||||
forHTTPHeaderField: "Authorization"
|
||||
)
|
||||
|
||||
let wsTask = session.webSocketTask(with: request)
|
||||
wsTask.resume()
|
||||
|
||||
try await withThrowingTaskGroup(of: Void.self) { group in
|
||||
let events = BailianEventStream(task: wsTask)
|
||||
|
||||
group.addTask {
|
||||
defer { events.cancel() }
|
||||
try await Self.sendText(
|
||||
Self.runTaskMessage(taskID: taskID, model: resolvedModel, vocabularyID: nil),
|
||||
task: wsTask
|
||||
)
|
||||
try await events.waitForStarted(timeout: Self.startTimeout)
|
||||
// Politely end the task; the connection is already proven.
|
||||
try? await Self.sendText(Self.finishTaskMessage(taskID: taskID), task: wsTask)
|
||||
}
|
||||
|
||||
group.addTask {
|
||||
try await Task.sleep(nanoseconds: UInt64(Self.startTimeout * 1_000_000_000))
|
||||
events.cancel()
|
||||
wsTask.cancel(with: .goingAway, reason: nil)
|
||||
throw CloudASRError.transport("connection probe timed out")
|
||||
}
|
||||
|
||||
_ = try await group.next()
|
||||
group.cancelAll()
|
||||
}
|
||||
}
|
||||
|
||||
private static func runSession(
|
||||
taskID: String,
|
||||
model: String,
|
||||
pcm: Data,
|
||||
wsTask: URLSessionWebSocketTask,
|
||||
events: BailianEventStream
|
||||
) async throws -> String {
|
||||
try await sendText(
|
||||
runTaskMessage(taskID: taskID, model: model, vocabularyID: nil),
|
||||
task: wsTask
|
||||
)
|
||||
|
||||
try await events.waitForStarted(timeout: startTimeout)
|
||||
|
||||
var offset = 0
|
||||
while offset < pcm.count {
|
||||
let end = min(offset + targetChunkBytes, pcm.count)
|
||||
try await sendBinary(pcm.subdata(in: offset..<end), task: wsTask)
|
||||
offset = end
|
||||
}
|
||||
|
||||
// Let the server register the final frames before ending the task.
|
||||
// Sending `finish-task` in the same instant as the last binary frame
|
||||
// races the server's audio buffering (root cause of `emptyAudio` on
|
||||
// very short clips).
|
||||
try? await Task.sleep(nanoseconds: 120_000_000)
|
||||
|
||||
try await sendText(finishTaskMessage(taskID: taskID), task: wsTask)
|
||||
return try await events.waitForFinalText(timeout: finalTimeout)
|
||||
}
|
||||
|
||||
private func resolvedEndpointURL() throws -> URL {
|
||||
let raw = endpoint.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? CloudASRModelCatalog.bailianDefaultEndpoint
|
||||
: endpoint.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
|
||||
return url
|
||||
}
|
||||
|
||||
private static func sendText(_ text: String, task: URLSessionWebSocketTask) async throws {
|
||||
do {
|
||||
try await task.send(.string(text))
|
||||
} catch {
|
||||
throw CloudASRError.transport(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private static func sendBinary(_ data: Data, task: URLSessionWebSocketTask) async throws {
|
||||
do {
|
||||
try await task.send(.data(data))
|
||||
} catch {
|
||||
throw CloudASRError.transport(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private static func pcm16Data(samples: [Float]) -> Data {
|
||||
var data = Data()
|
||||
data.reserveCapacity(samples.count * 2)
|
||||
for sample in samples {
|
||||
let scaled = sample * 32_767.0
|
||||
let clipped = Swift.max(-32_768.0, Swift.min(32_767.0, scaled))
|
||||
var littleEndian = Int16(clipped.rounded()).littleEndian
|
||||
withUnsafeBytes(of: &littleEndian) { data.append(contentsOf: $0) }
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
/// Overlap-aware join to avoid cumulative duplicate text from interim replays.
|
||||
static func mergeSegments(_ segments: [String]) -> String {
|
||||
var result = ""
|
||||
for segment in segments {
|
||||
if result.isEmpty {
|
||||
result = segment
|
||||
continue
|
||||
}
|
||||
let resultChars = Array(result)
|
||||
let segmentChars = Array(segment)
|
||||
let maxOverlap = min(resultChars.count, segmentChars.count)
|
||||
var overlap = 0
|
||||
if maxOverlap >= 2 {
|
||||
for length in stride(from: maxOverlap, through: 2, by: -1) {
|
||||
let tail = resultChars.suffix(length)
|
||||
let head = segmentChars.prefix(length)
|
||||
if tail.elementsEqual(head) {
|
||||
overlap = length
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
result.append(contentsOf: segmentChars.dropFirst(overlap))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
static func runTaskMessage(taskID: String, model: String, vocabularyID: String?) -> String {
|
||||
var parameters: [String: Any] = [
|
||||
"sample_rate": 16_000,
|
||||
"format": "pcm",
|
||||
]
|
||||
if let vocabularyID = vocabularyID?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!vocabularyID.isEmpty {
|
||||
parameters["vocabulary_id"] = vocabularyID
|
||||
}
|
||||
let body: [String: Any] = [
|
||||
"header": [
|
||||
"action": "run-task",
|
||||
"task_id": taskID,
|
||||
"streaming": "duplex",
|
||||
],
|
||||
"payload": [
|
||||
"task_group": "audio",
|
||||
"task": "asr",
|
||||
"function": "recognition",
|
||||
"model": model,
|
||||
"parameters": parameters,
|
||||
"input": [:] as [String: Any],
|
||||
],
|
||||
]
|
||||
guard let data = try? JSONSerialization.data(withJSONObject: body),
|
||||
let json = String(data: data, encoding: .utf8) else {
|
||||
return "{}"
|
||||
}
|
||||
return json
|
||||
}
|
||||
|
||||
static func finishTaskMessage(taskID: String) -> String {
|
||||
let body: [String: Any] = [
|
||||
"header": [
|
||||
"action": "finish-task",
|
||||
"task_id": taskID,
|
||||
"streaming": "duplex",
|
||||
],
|
||||
"payload": ["input": [:] as [String: Any]],
|
||||
]
|
||||
guard let data = try? JSONSerialization.data(withJSONObject: body),
|
||||
let json = String(data: data, encoding: .utf8) else {
|
||||
return "{}"
|
||||
}
|
||||
return json
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Concurrent read loop
|
||||
|
||||
private final class BailianEventStream: @unchecked Sendable {
|
||||
private let task: URLSessionWebSocketTask
|
||||
private let lock = NSLock()
|
||||
private var started = false
|
||||
private var finalText: String?
|
||||
private var failure: Error?
|
||||
private var readTask: Task<Void, Never>?
|
||||
|
||||
init(task: URLSessionWebSocketTask) {
|
||||
self.task = task
|
||||
readTask = Task { [weak self] in
|
||||
await self?.readLoop()
|
||||
}
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
readTask?.cancel()
|
||||
task.cancel(with: .goingAway, reason: nil)
|
||||
}
|
||||
|
||||
func waitForStarted(timeout: TimeInterval) async throws {
|
||||
let deadline = Date().addingTimeInterval(timeout)
|
||||
while Date() < deadline {
|
||||
if let failure = snapshotFailure() { throw failure }
|
||||
if snapshotStarted() { return }
|
||||
try await Task.sleep(nanoseconds: 20_000_000)
|
||||
}
|
||||
cancel()
|
||||
throw CloudASRError.transport("task-started timed out")
|
||||
}
|
||||
|
||||
func waitForFinalText(timeout: TimeInterval) async throws -> String {
|
||||
let deadline = Date().addingTimeInterval(timeout)
|
||||
while Date() < deadline {
|
||||
if let failure = snapshotFailure() { throw failure }
|
||||
if let text = snapshotFinalText() { return text }
|
||||
try await Task.sleep(nanoseconds: 20_000_000)
|
||||
}
|
||||
cancel()
|
||||
throw CloudASRError.transport("final result timed out")
|
||||
}
|
||||
|
||||
private func snapshotStarted() -> Bool {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return started
|
||||
}
|
||||
|
||||
private func snapshotFinalText() -> String? {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return finalText
|
||||
}
|
||||
|
||||
private func snapshotFailure() -> Error? {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return failure
|
||||
}
|
||||
|
||||
private func readLoop() async {
|
||||
var finalSegments: [Int64: String] = [:]
|
||||
var partialSegments: [Int64: String] = [:]
|
||||
var lastResultText = ""
|
||||
|
||||
while !Task.isCancelled {
|
||||
let message: URLSessionWebSocketTask.Message
|
||||
do {
|
||||
message = try await task.receive()
|
||||
} catch {
|
||||
publishFailure(CloudASRError.transport(error.localizedDescription))
|
||||
return
|
||||
}
|
||||
|
||||
let text: String
|
||||
switch message {
|
||||
case .string(let value):
|
||||
text = value
|
||||
case .data(let data):
|
||||
text = String(data: data, encoding: .utf8) ?? ""
|
||||
@unknown default:
|
||||
continue
|
||||
}
|
||||
guard !text.isEmpty else { continue }
|
||||
|
||||
guard let json = try? JSONSerialization.jsonObject(with: Data(text.utf8)) as? [String: Any],
|
||||
let header = json["header"] as? [String: Any] else {
|
||||
continue
|
||||
}
|
||||
let event = header["event"] as? String ?? ""
|
||||
|
||||
switch event {
|
||||
case "task-started":
|
||||
publishStarted()
|
||||
case "result-generated":
|
||||
guard let payload = json["payload"] as? [String: Any],
|
||||
let output = payload["output"] as? [String: Any],
|
||||
let sentenceObj = output["sentence"] as? [String: Any] else {
|
||||
continue
|
||||
}
|
||||
if sentenceObj["heartbeat"] as? Bool == true { continue }
|
||||
guard let rawText = sentenceObj["text"] as? String else { continue }
|
||||
let trimmed = rawText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { continue }
|
||||
|
||||
lastResultText = trimmed
|
||||
let sentenceID = sentenceObj["sentence_id"] as? Int64 ?? 0
|
||||
let sentenceEndValue = sentenceObj["sentence_end"]
|
||||
let sentenceEnd = sentenceEndValue as? Bool ?? false
|
||||
let endTime = sentenceObj["end_time"] as? Int64 ?? 0
|
||||
let isFinal = sentenceEndValue != nil ? sentenceEnd : endTime > 0
|
||||
|
||||
if isFinal {
|
||||
finalSegments[sentenceID] = trimmed
|
||||
partialSegments.removeValue(forKey: sentenceID)
|
||||
} else {
|
||||
partialSegments[sentenceID] = trimmed
|
||||
}
|
||||
case "task-finished":
|
||||
if finalSegments.isEmpty {
|
||||
publishFinal(lastResultText)
|
||||
} else {
|
||||
let ordered = finalSegments.keys.sorted().compactMap { finalSegments[$0] }
|
||||
publishFinal(BailianRealtimeASRClient.mergeSegments(ordered))
|
||||
}
|
||||
return
|
||||
case "task-failed":
|
||||
let message = header["error_message"] as? String ?? "task failed"
|
||||
publishFailure(CloudASRError.transport(message))
|
||||
return
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func publishStarted() {
|
||||
lock.lock()
|
||||
started = true
|
||||
lock.unlock()
|
||||
}
|
||||
|
||||
private func publishFinal(_ text: String) {
|
||||
lock.lock()
|
||||
finalText = text
|
||||
lock.unlock()
|
||||
}
|
||||
|
||||
private func publishFailure(_ error: Error) {
|
||||
lock.lock()
|
||||
failure = error
|
||||
lock.unlock()
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,28 @@ public protocol CloudASRTranscribing: Sendable {
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String
|
||||
|
||||
/// Settings "validate connection" probe. Verifies transport + auth only.
|
||||
func probeConnection() async throws
|
||||
}
|
||||
|
||||
extension CloudASRTranscribing {
|
||||
/// Default probe: transcribe ~1 s of near-silence. An empty transcript
|
||||
/// counts as success — HTTP/streaming providers only need to prove that
|
||||
/// transport + auth work. Providers whose service rejects silent/short
|
||||
/// audio (e.g. DashScope realtime returns `emptyAudio`) override this.
|
||||
public func probeConnection() async throws {
|
||||
do {
|
||||
_ = try await transcribe(
|
||||
samples: [Float](repeating: 0.01, count: 16_000),
|
||||
sampleRate: 16_000,
|
||||
locale: Locale(identifier: "zh-CN"),
|
||||
dictionary: .empty
|
||||
)
|
||||
} catch CloudASRError.emptyTranscript {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public enum CloudASRClientFactory {
|
||||
@@ -29,11 +51,12 @@ public enum CloudASRClientFactory {
|
||||
model: asrModel,
|
||||
session: session
|
||||
)
|
||||
case .alibabaVocabulary:
|
||||
return AlibabaFunASRClient(
|
||||
case .bailianStreaming:
|
||||
return BailianRealtimeASRClient(
|
||||
apiKey: store.asrApiKey,
|
||||
endpoint: store.asrBaseURL,
|
||||
model: asrModel,
|
||||
persistence: store.cloudASRPersistence,
|
||||
vocabularyID: nil,
|
||||
session: session
|
||||
)
|
||||
case .prompt:
|
||||
@@ -44,6 +67,22 @@ public enum CloudASRClientFactory {
|
||||
model: asrModel,
|
||||
session: session
|
||||
)
|
||||
case .openRouterJson:
|
||||
return PromptCloudASRClient(
|
||||
providerId: providerId,
|
||||
baseURL: store.asrBaseURL,
|
||||
apiKey: store.asrApiKey,
|
||||
model: asrModel,
|
||||
session: session,
|
||||
requestFormat: .openRouterJson
|
||||
)
|
||||
case .volcengineStreaming:
|
||||
return VolcengineCloudASRClient(
|
||||
apiKey: store.asrApiKey,
|
||||
endpoint: store.asrBaseURL,
|
||||
resourceID: asrModel,
|
||||
session: session
|
||||
)
|
||||
case .localFallback:
|
||||
return UnsupportedCloudASRClient(providerId: providerId)
|
||||
}
|
||||
@@ -151,25 +190,14 @@ struct ZhipuCloudASRClient: CloudASRTranscribing {
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Alibaba Fun-ASR Flash (vocabulary_id + context)
|
||||
// MARK: - Alibaba Fun-ASR Flash (HTTP sync, context text bias)
|
||||
|
||||
/// `UserDefaults` is not `Sendable`; we only touch `persistence` on the
|
||||
/// actor-isolated cloud ASR path, same as the previous `AppGroupStore` holder.
|
||||
struct AlibabaFunASRClient: CloudASRTranscribing, @unchecked Sendable {
|
||||
struct AlibabaFunASRClient: CloudASRTranscribing {
|
||||
let apiKey: String
|
||||
let model: String
|
||||
let persistence: UserDefaults
|
||||
let session: URLSession
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {
|
||||
_ = try await AlibabaVocabularySync.ensureVocabularyID(
|
||||
dictionary: dictionary,
|
||||
apiKey: apiKey,
|
||||
targetModel: CloudASRModelCatalog.alibabaVocabularyTargetModel,
|
||||
defaults: persistence,
|
||||
session: session
|
||||
)
|
||||
}
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
@@ -179,14 +207,6 @@ struct AlibabaFunASRClient: CloudASRTranscribing, @unchecked Sendable {
|
||||
) async throws -> String {
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
|
||||
let vocabularyID = try await AlibabaVocabularySync.ensureVocabularyID(
|
||||
dictionary: dictionary,
|
||||
apiKey: apiKey,
|
||||
targetModel: CloudASRModelCatalog.alibabaVocabularyTargetModel,
|
||||
defaults: persistence,
|
||||
session: session
|
||||
)
|
||||
|
||||
let dataURI = PCMSampleWavEncoder.dataURI(samples: samples, sampleRate: sampleRate)
|
||||
let urlString = CloudASRModelCatalog.alibabaAPIBase + CloudASRModelCatalog.alibabaMultimodalPath
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
@@ -211,13 +231,10 @@ struct AlibabaFunASRClient: CloudASRTranscribing, @unchecked Sendable {
|
||||
],
|
||||
])
|
||||
|
||||
var parameters: [String: Any] = [
|
||||
let parameters: [String: Any] = [
|
||||
"format": "wav",
|
||||
"sample_rate": "\(sampleRate)",
|
||||
]
|
||||
if let vocabularyID, !vocabularyID.isEmpty {
|
||||
parameters["vocabulary_id"] = vocabularyID
|
||||
}
|
||||
|
||||
let body: [String: Any] = [
|
||||
"model": model,
|
||||
@@ -235,11 +252,11 @@ struct AlibabaFunASRClient: CloudASRTranscribing, @unchecked Sendable {
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
||||
guard let text = Self.parseText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty else {
|
||||
throw CloudASRError.emptyTranscript
|
||||
if let text = Self.parseText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty {
|
||||
return text
|
||||
}
|
||||
return text
|
||||
return ""
|
||||
}
|
||||
|
||||
private static func parseText(from data: Data) -> String? {
|
||||
@@ -256,7 +273,13 @@ struct AlibabaFunASRClient: CloudASRTranscribing, @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Prompt-biased transcription (OpenAI / MiMo / custom)
|
||||
// MARK: - Prompt-biased transcription (OpenAI / MiMo / Groq / custom)
|
||||
|
||||
enum PromptCloudASRRequestFormat: Sendable {
|
||||
case multipart
|
||||
/// OpenRouter expects JSON `{ model, input_audio: { data, format } }`.
|
||||
case openRouterJson
|
||||
}
|
||||
|
||||
struct PromptCloudASRClient: CloudASRTranscribing {
|
||||
let providerId: String
|
||||
@@ -264,6 +287,26 @@ struct PromptCloudASRClient: CloudASRTranscribing {
|
||||
let apiKey: String
|
||||
let model: String
|
||||
let session: URLSession
|
||||
var requestFormat: PromptCloudASRRequestFormat = .multipart
|
||||
|
||||
/// Groq / OpenRouter batch uploads cap around 30 s per request.
|
||||
private static let whisperCompatibleMaxDurationSeconds: TimeInterval = 30
|
||||
|
||||
init(
|
||||
providerId: String,
|
||||
baseURL: String,
|
||||
apiKey: String,
|
||||
model: String,
|
||||
session: URLSession,
|
||||
requestFormat: PromptCloudASRRequestFormat = .multipart
|
||||
) {
|
||||
self.providerId = providerId
|
||||
self.baseURL = baseURL
|
||||
self.apiKey = apiKey
|
||||
self.model = model
|
||||
self.session = session
|
||||
self.requestFormat = requestFormat
|
||||
}
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
@@ -281,6 +324,13 @@ struct PromptCloudASRClient: CloudASRTranscribing {
|
||||
dictionary: dictionary
|
||||
)
|
||||
}
|
||||
if requestFormat == .openRouterJson {
|
||||
return try await transcribeOpenRouterJSON(
|
||||
samples: samples,
|
||||
sampleRate: sampleRate,
|
||||
dictionary: dictionary
|
||||
)
|
||||
}
|
||||
return try await transcribeOpenAIStyle(
|
||||
samples: samples,
|
||||
sampleRate: sampleRate,
|
||||
@@ -288,11 +338,21 @@ struct PromptCloudASRClient: CloudASRTranscribing {
|
||||
)
|
||||
}
|
||||
|
||||
private func enforceWhisperDuration(samples: [Float], sampleRate: Int) throws {
|
||||
let duration = Double(samples.count) / Double(sampleRate)
|
||||
guard duration <= Self.whisperCompatibleMaxDurationSeconds else {
|
||||
throw CloudASRError.audioTooLong
|
||||
}
|
||||
}
|
||||
|
||||
private func transcribeOpenAIStyle(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
if providerId == "groq" || providerId == "openai" || providerId == "custom" {
|
||||
try enforceWhisperDuration(samples: samples, sampleRate: sampleRate)
|
||||
}
|
||||
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
|
||||
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
|
||||
let urlString = "\(trimmedBase)/audio/transcriptions"
|
||||
@@ -335,6 +395,45 @@ struct PromptCloudASRClient: CloudASRTranscribing {
|
||||
return text
|
||||
}
|
||||
|
||||
private func transcribeOpenRouterJSON(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
try enforceWhisperDuration(samples: samples, sampleRate: sampleRate)
|
||||
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
|
||||
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
|
||||
let urlString = "\(trimmedBase)/audio/transcriptions"
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
|
||||
var body: [String: Any] = [
|
||||
"model": model,
|
||||
"input_audio": [
|
||||
"data": wav.base64EncodedString(),
|
||||
"format": "wav",
|
||||
],
|
||||
]
|
||||
let prompt = dictionary.asrPromptBias(maxCharacters: 600)
|
||||
if !prompt.isEmpty {
|
||||
body["prompt"] = prompt
|
||||
}
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
||||
request.httpBody = try JSONSerialization.data(withJSONObject: body)
|
||||
request.timeoutInterval = 90
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
||||
guard let text = Self.parseOpenAIText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty else {
|
||||
throw CloudASRError.emptyTranscript
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
private func transcribeMiMo(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
// CloudASRConnectionCheck.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Settings "validate connection" probe shared by iOS and macOS.
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum CloudASRConnectionCheck {
|
||||
/// Verifies the active cloud ASR client can connect + authenticate.
|
||||
///
|
||||
/// Each backend decides how to probe (see `CloudASRTranscribing`):
|
||||
/// HTTP/batch providers transcribe a short silence clip and treat an
|
||||
/// empty transcript as success; DashScope realtime only handshakes to
|
||||
/// `task-started` (pushing fake audio makes it fail with `emptyAudio`).
|
||||
public static func validate(store: any ConfigurationStore) async throws {
|
||||
let client = CloudASRClientFactory.make(store: store)
|
||||
try await client.probeConnection()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
// VolcengineCloudASRClient.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Volcengine SAUC bigmodel ASR client. The service uses a WebSocket with a
|
||||
// small custom binary frame wrapper; this file keeps that protocol isolated
|
||||
// from the HTTP-style cloud ASR clients.
|
||||
|
||||
import Foundation
|
||||
|
||||
struct VolcengineCloudASRClient: CloudASRTranscribing {
|
||||
let apiKey: String
|
||||
let endpoint: String
|
||||
let resourceID: String
|
||||
let session: URLSession
|
||||
|
||||
private static let targetChunkBytes = 6_400 // 200 ms @ 16 kHz, 16-bit, mono.
|
||||
private static let finalTimeout: TimeInterval = 12
|
||||
private static let hotwordCap = 80
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
guard !samples.isEmpty else { throw CloudASRError.emptyTranscript }
|
||||
let credentials = try VolcengineCredentials.parse(
|
||||
apiKey: apiKey,
|
||||
fallbackResourceID: resolvedResourceID
|
||||
)
|
||||
let url = try resolvedEndpointURL()
|
||||
let pcm = Self.pcm16Data(samples: samples)
|
||||
let connectID = UUID().uuidString
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.timeoutInterval = 8
|
||||
request.setValue(credentials.appID, forHTTPHeaderField: "X-Api-App-Key")
|
||||
request.setValue(credentials.accessToken, forHTTPHeaderField: "X-Api-Access-Key")
|
||||
request.setValue(credentials.resourceID, forHTTPHeaderField: "X-Api-Resource-Id")
|
||||
request.setValue(connectID, forHTTPHeaderField: "X-Api-Connect-Id")
|
||||
|
||||
let task = session.webSocketTask(with: request)
|
||||
task.resume()
|
||||
defer {
|
||||
task.cancel(with: .normalClosure, reason: nil)
|
||||
}
|
||||
|
||||
let firstPayload = try Self.firstFramePayload(connectID: connectID, dictionary: dictionary)
|
||||
try await send(
|
||||
VolcengineFrame.build(
|
||||
messageType: .fullClientRequest,
|
||||
flags: .positiveSequence,
|
||||
serialization: .json,
|
||||
payload: firstPayload,
|
||||
sequence: 1
|
||||
),
|
||||
task: task
|
||||
)
|
||||
|
||||
var sequence = 2
|
||||
var offset = 0
|
||||
while offset < pcm.count {
|
||||
let end = min(offset + Self.targetChunkBytes, pcm.count)
|
||||
try await send(
|
||||
VolcengineFrame.build(
|
||||
messageType: .audioOnlyRequest,
|
||||
flags: .positiveSequence,
|
||||
serialization: .none,
|
||||
payload: pcm.subdata(in: offset..<end),
|
||||
sequence: Int32(sequence)
|
||||
),
|
||||
task: task
|
||||
)
|
||||
sequence += 1
|
||||
offset = end
|
||||
}
|
||||
|
||||
try await send(
|
||||
VolcengineFrame.build(
|
||||
messageType: .audioOnlyRequest,
|
||||
flags: .negativeSequence,
|
||||
serialization: .none,
|
||||
payload: Data(),
|
||||
sequence: -Int32(sequence)
|
||||
),
|
||||
task: task
|
||||
)
|
||||
|
||||
let text = try await receiveFinalText(task: task)
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { throw CloudASRError.emptyTranscript }
|
||||
return trimmed
|
||||
}
|
||||
|
||||
private var resolvedResourceID: String {
|
||||
resourceID.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? CloudASRModelCatalog.volcengineDefaultResourceID
|
||||
: resourceID.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
|
||||
private func resolvedEndpointURL() throws -> URL {
|
||||
let raw = endpoint.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
? CloudASRModelCatalog.volcengineEndpoint
|
||||
: endpoint.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
|
||||
return url
|
||||
}
|
||||
|
||||
private func send(_ data: Data, task: URLSessionWebSocketTask) async throws {
|
||||
do {
|
||||
try await task.send(.data(data))
|
||||
} catch {
|
||||
throw CloudASRError.transport(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private func receiveFinalText(task: URLSessionWebSocketTask) async throws -> String {
|
||||
try await withThrowingTaskGroup(of: String.self) { group in
|
||||
group.addTask {
|
||||
var lastPartial = ""
|
||||
while true {
|
||||
let message = try await task.receive()
|
||||
let data: Data
|
||||
switch message {
|
||||
case .data(let payload):
|
||||
data = payload
|
||||
case .string(let string):
|
||||
data = Data(string.utf8)
|
||||
@unknown default:
|
||||
continue
|
||||
}
|
||||
|
||||
guard let frame = VolcengineFrame.parse(data) else { continue }
|
||||
if frame.messageType == .errorMessage {
|
||||
let body = String(data: frame.payload, encoding: .utf8) ?? ""
|
||||
let code = frame.errorCode ?? 0
|
||||
throw CloudASRError.transport("ASR error \(code): \(body)")
|
||||
}
|
||||
guard frame.messageType == .fullServerResponse else { continue }
|
||||
let parsedText = Self.text(from: frame.payload)
|
||||
if !parsedText.isEmpty {
|
||||
lastPartial = parsedText
|
||||
}
|
||||
if frame.isFinal {
|
||||
return parsedText.isEmpty ? lastPartial : parsedText
|
||||
}
|
||||
}
|
||||
}
|
||||
group.addTask {
|
||||
try await Task.sleep(nanoseconds: UInt64(Self.finalTimeout * 1_000_000_000))
|
||||
throw CloudASRError.transport("Volcengine final result timed out")
|
||||
}
|
||||
let result = try await group.next()!
|
||||
group.cancelAll()
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
private static func firstFramePayload(
|
||||
connectID: String,
|
||||
dictionary: PersonalDictionary
|
||||
) throws -> Data {
|
||||
var request: [String: Any] = [
|
||||
"model_name": "bigmodel",
|
||||
"enable_itn": true,
|
||||
"enable_punc": true,
|
||||
"show_utterances": true,
|
||||
"enable_speaker_info": true,
|
||||
]
|
||||
if let context = hotwordContext(dictionary: dictionary) {
|
||||
request["context"] = context
|
||||
}
|
||||
|
||||
let payload: [String: Any] = [
|
||||
"user": ["uid": connectID],
|
||||
"audio": [
|
||||
"format": "pcm",
|
||||
"rate": 16_000,
|
||||
"bits": 16,
|
||||
"channel": 1,
|
||||
"codec": "raw",
|
||||
],
|
||||
"request": request,
|
||||
]
|
||||
return try JSONSerialization.data(withJSONObject: payload)
|
||||
}
|
||||
|
||||
private static func hotwordContext(dictionary: PersonalDictionary) -> String? {
|
||||
var seen: [String] = []
|
||||
for word in dictionary.asrHotwords() {
|
||||
let trimmed = word.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { continue }
|
||||
guard !seen.contains(where: { $0.caseInsensitiveCompare(trimmed) == .orderedSame }) else {
|
||||
continue
|
||||
}
|
||||
seen.append(trimmed)
|
||||
if seen.count >= hotwordCap { break }
|
||||
}
|
||||
guard !seen.isEmpty else { return nil }
|
||||
let words = seen.map { ["word": $0] }
|
||||
guard let data = try? JSONSerialization.data(withJSONObject: ["hotwords": words]) else {
|
||||
return nil
|
||||
}
|
||||
return String(data: data, encoding: .utf8)
|
||||
}
|
||||
|
||||
private static func pcm16Data(samples: [Float]) -> Data {
|
||||
var data = Data()
|
||||
data.reserveCapacity(samples.count * 2)
|
||||
for sample in samples {
|
||||
let scaled = sample * 32_767.0
|
||||
let clipped = Swift.max(-32_768.0, Swift.min(32_767.0, scaled))
|
||||
var littleEndian = Int16(clipped.rounded()).littleEndian
|
||||
withUnsafeBytes(of: &littleEndian) { data.append(contentsOf: $0) }
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
private static func text(from payload: Data) -> String {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: payload) as? [String: Any],
|
||||
let result = normalizedResult(from: json) else {
|
||||
return ""
|
||||
}
|
||||
|
||||
if let utterances = result["utterances"] as? [[String: Any]], !utterances.isEmpty {
|
||||
let pieces = utterances.compactMap { $0["text"] as? String }
|
||||
let joined = pieces.joined()
|
||||
if !joined.isEmpty { return joined }
|
||||
}
|
||||
return result["text"] as? String ?? ""
|
||||
}
|
||||
|
||||
private static func normalizedResult(from json: [String: Any]) -> [String: Any]? {
|
||||
if let result = json["result"] as? [String: Any] {
|
||||
return result
|
||||
}
|
||||
if let results = json["result"] as? [[String: Any]] {
|
||||
return results.first
|
||||
}
|
||||
if json["text"] as? String != nil {
|
||||
return json
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
private struct VolcengineCredentials {
|
||||
let appID: String
|
||||
let accessToken: String
|
||||
let resourceID: String
|
||||
|
||||
static func parse(apiKey: String, fallbackResourceID: String) throws -> VolcengineCredentials {
|
||||
let trimmed = apiKey.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
|
||||
if let data = trimmed.data(using: .utf8),
|
||||
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {
|
||||
let appID = string(json, keys: ["app_id", "appId", "appid"])
|
||||
let token = string(json, keys: ["access_token", "accessToken", "token"])
|
||||
let resourceID = string(json, keys: ["resource_id", "resourceId", "resource"])
|
||||
?? fallbackResourceID
|
||||
guard let appID, let token, !resourceID.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
return VolcengineCredentials(appID: appID, accessToken: token, resourceID: resourceID)
|
||||
}
|
||||
|
||||
let separators = CharacterSet(charactersIn: ":\n,")
|
||||
let parts = trimmed
|
||||
.components(separatedBy: separators)
|
||||
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
|
||||
.filter { !$0.isEmpty }
|
||||
guard parts.count >= 2 else { throw CloudASRError.noAPIKey }
|
||||
let resourceID = parts.count >= 3 ? parts[2] : fallbackResourceID
|
||||
return VolcengineCredentials(appID: parts[0], accessToken: parts[1], resourceID: resourceID)
|
||||
}
|
||||
|
||||
private static func string(_ json: [String: Any], keys: [String]) -> String? {
|
||||
for key in keys {
|
||||
if let value = json[key] as? String {
|
||||
let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !trimmed.isEmpty { return trimmed }
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
private enum VolcengineMessageType: UInt8 {
|
||||
case fullClientRequest = 0b0001
|
||||
case audioOnlyRequest = 0b0010
|
||||
case fullServerResponse = 0b1001
|
||||
case errorMessage = 0b1111
|
||||
}
|
||||
|
||||
private enum VolcengineFlags: UInt8 {
|
||||
case none = 0b0000
|
||||
case positiveSequence = 0b0001
|
||||
case lastPacket = 0b0010
|
||||
case negativeSequence = 0b0011
|
||||
}
|
||||
|
||||
private enum VolcengineSerialization: UInt8 {
|
||||
case none = 0b0000
|
||||
case json = 0b0001
|
||||
}
|
||||
|
||||
private struct VolcengineFrame {
|
||||
let messageType: VolcengineMessageType?
|
||||
let flags: UInt8
|
||||
let sequence: Int32?
|
||||
let errorCode: UInt32?
|
||||
let payload: Data
|
||||
|
||||
var isFinal: Bool {
|
||||
flags == VolcengineFlags.lastPacket.rawValue
|
||||
|| flags == VolcengineFlags.negativeSequence.rawValue
|
||||
|| (sequence ?? 0) < 0
|
||||
}
|
||||
|
||||
static func build(
|
||||
messageType: VolcengineMessageType,
|
||||
flags: VolcengineFlags,
|
||||
serialization: VolcengineSerialization,
|
||||
payload: Data,
|
||||
sequence: Int32?
|
||||
) -> Data {
|
||||
var data = Data()
|
||||
data.append(0x11)
|
||||
data.append((messageType.rawValue << 4) | flags.rawValue)
|
||||
data.append(serialization.rawValue << 4)
|
||||
data.append(0x00)
|
||||
|
||||
if flags == .positiveSequence || flags == .negativeSequence, let sequence {
|
||||
data.appendBE32(UInt32(bitPattern: sequence))
|
||||
}
|
||||
data.appendBE32(UInt32(payload.count))
|
||||
data.append(payload)
|
||||
return data
|
||||
}
|
||||
|
||||
static func parse(_ data: Data) -> VolcengineFrame? {
|
||||
guard data.count >= 8 else { return nil }
|
||||
let bytes = [UInt8](data)
|
||||
let headerSize = Int(bytes[0] & 0x0F) * 4
|
||||
guard headerSize >= 4, data.count >= headerSize + 4 else { return nil }
|
||||
|
||||
let typeRaw = (bytes[1] >> 4) & 0x0F
|
||||
let messageType = VolcengineMessageType(rawValue: typeRaw)
|
||||
let flags = bytes[1] & 0x0F
|
||||
let compression = bytes[2] & 0x0F
|
||||
guard compression == 0 else { return nil }
|
||||
|
||||
var offset = headerSize
|
||||
var sequence: Int32?
|
||||
if flags == VolcengineFlags.positiveSequence.rawValue
|
||||
|| flags == VolcengineFlags.negativeSequence.rawValue {
|
||||
guard let value = data.readBE32(at: offset) else { return nil }
|
||||
sequence = Int32(bitPattern: value)
|
||||
offset += 4
|
||||
}
|
||||
|
||||
if messageType == .errorMessage {
|
||||
guard let code = data.readBE32(at: offset),
|
||||
let size = data.readBE32(at: offset + 4) else { return nil }
|
||||
offset += 8
|
||||
guard data.count >= offset + Int(size) else { return nil }
|
||||
return VolcengineFrame(
|
||||
messageType: messageType,
|
||||
flags: flags,
|
||||
sequence: sequence,
|
||||
errorCode: code,
|
||||
payload: data.subdata(in: offset..<(offset + Int(size)))
|
||||
)
|
||||
}
|
||||
|
||||
guard let size = data.readBE32(at: offset) else { return nil }
|
||||
offset += 4
|
||||
guard data.count >= offset + Int(size) else { return nil }
|
||||
return VolcengineFrame(
|
||||
messageType: messageType,
|
||||
flags: flags,
|
||||
sequence: sequence,
|
||||
errorCode: nil,
|
||||
payload: data.subdata(in: offset..<(offset + Int(size)))
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private extension Data {
|
||||
mutating func appendBE32(_ value: UInt32) {
|
||||
var bigEndian = value.bigEndian
|
||||
Swift.withUnsafeBytes(of: &bigEndian) { append(contentsOf: $0) }
|
||||
}
|
||||
|
||||
func readBE32(at offset: Int) -> UInt32? {
|
||||
guard count >= offset + 4 else { return nil }
|
||||
return self[offset..<(offset + 4)].reduce(UInt32(0)) { ($0 << 8) | UInt32($1) }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user