Rocky
c2f07bd8d2
Introduce a standalone macOS menu-bar app (OSGKeyboardMac) that reuses the platform-agnostic OSGKeyboardShared core: record -> cloud/local ASR -> polish -> insert. Local mode uses Qwen3-ASR via mlx-swift-asr (macOS 15+, Apple Silicon); iOS targets stay zero-SPM. Harden iCloud sync for multi-device correctness: - Per-field settings merge (appSettings.v2) so concurrent edits no longer clobber each other's unrelated fields. - Per-device usage statistics (G-Counter) that sum instead of max(). - Tombstoned dictionary/history merge so deletes propagate and entries can't resurrect. - API keys replicate via iCloud Keychain, never iCloud KVS JSON; pulling a legacy blob without key fields no longer wipes local Keychain entries. - Add a low-risk "Sync Now" action in Settings. Fix Flow keyboard mic state: stay orange until the host publishes a real ready contract, share a single MicVoiceAvailability gate, and self-heal stale cross-process heartbeat jitter instead of getting stuck. Extract shared storage (SpeechHistoryStore/UsageStatisticsStore, ConfigurationStore) into OSGKeyboardShared and add tests for the new sync/merge logic.
412 lines
15 KiB
Swift
412 lines
15 KiB
Swift
// CloudASRClients.swift
|
|
// OSGKeyboard · Shared
|
|
//
|
|
// Provider-specific cloud ASR backends with personal-dictionary bias.
|
|
|
|
import Foundation
|
|
|
|
public protocol CloudASRTranscribing: Sendable {
|
|
func prepare(dictionary: PersonalDictionary) async throws
|
|
func transcribe(
|
|
samples: [Float],
|
|
sampleRate: Int,
|
|
locale: Locale,
|
|
dictionary: PersonalDictionary
|
|
) async throws -> String
|
|
}
|
|
|
|
public enum CloudASRClientFactory {
|
|
public static func make(store: any ConfigurationStore, session: URLSession = .shared) -> CloudASRTranscribing {
|
|
let strategy = CloudASRModelCatalog.strategy(for: store.providerId)
|
|
switch strategy {
|
|
case .zhipuHotwords:
|
|
return ZhipuCloudASRClient(
|
|
apiKey: store.apiKey,
|
|
model: CloudASRModelCatalog.defaultModel(for: store.providerId),
|
|
session: session
|
|
)
|
|
case .alibabaVocabulary:
|
|
return AlibabaFunASRClient(
|
|
apiKey: store.apiKey,
|
|
model: CloudASRModelCatalog.defaultModel(for: store.providerId),
|
|
persistence: store.cloudASRPersistence,
|
|
session: session
|
|
)
|
|
case .prompt:
|
|
return PromptCloudASRClient(
|
|
providerId: store.providerId,
|
|
baseURL: store.baseURL,
|
|
apiKey: store.apiKey,
|
|
model: CloudASRModelCatalog.defaultModel(for: store.providerId),
|
|
session: session
|
|
)
|
|
case .localFallback:
|
|
return UnsupportedCloudASRClient(providerId: store.providerId)
|
|
}
|
|
}
|
|
}
|
|
|
|
// MARK: - Zhipu (hotwords + prompt)
|
|
|
|
struct ZhipuCloudASRClient: CloudASRTranscribing {
|
|
let apiKey: String
|
|
let model: String
|
|
let session: URLSession
|
|
|
|
private static let maxDurationSeconds: TimeInterval = 30
|
|
|
|
func prepare(dictionary: PersonalDictionary) async throws {}
|
|
|
|
func transcribe(
|
|
samples: [Float],
|
|
sampleRate: Int,
|
|
locale: Locale,
|
|
dictionary: PersonalDictionary
|
|
) async throws -> String {
|
|
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
|
let duration = Double(samples.count) / Double(sampleRate)
|
|
guard duration <= Self.maxDurationSeconds else { throw CloudASRError.audioTooLong }
|
|
|
|
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
|
|
let urlString = "https://open.bigmodel.cn/api/paas/v4\(CloudASRModelCatalog.zhipuTranscriptionPath)"
|
|
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
|
|
|
let boundary = "Boundary-\(UUID().uuidString)"
|
|
var body = Data()
|
|
func appendField(_ name: String, _ value: String) {
|
|
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
|
body.append("Content-Disposition: form-data; name=\"\(name)\"\r\n\r\n".data(using: .utf8)!)
|
|
body.append("\(value)\r\n".data(using: .utf8)!)
|
|
}
|
|
|
|
appendField("model", model)
|
|
appendField("stream", "false")
|
|
|
|
let hotwords = dictionary.asrHotwords()
|
|
if !hotwords.isEmpty,
|
|
let hotwordsJSON = try? JSONSerialization.data(withJSONObject: hotwords),
|
|
let hotwordsString = String(data: hotwordsJSON, encoding: .utf8) {
|
|
appendField("hotwords", hotwordsString)
|
|
}
|
|
|
|
let prompt = dictionary.asrPromptBias()
|
|
if !prompt.isEmpty {
|
|
appendField("prompt", prompt)
|
|
}
|
|
|
|
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
|
body.append("Content-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\n".data(using: .utf8)!)
|
|
body.append("Content-Type: audio/wav\r\n\r\n".data(using: .utf8)!)
|
|
body.append(wav)
|
|
body.append("\r\n".data(using: .utf8)!)
|
|
body.append("--\(boundary)--\r\n".data(using: .utf8)!)
|
|
|
|
var request = URLRequest(url: url)
|
|
request.httpMethod = "POST"
|
|
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
|
request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")
|
|
request.httpBody = body
|
|
request.timeoutInterval = 60
|
|
|
|
let (data, response) = try await session.data(for: request)
|
|
try Self.validateHTTP(response: response, data: data)
|
|
guard let text = Self.parseZhipuText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
|
!text.isEmpty else {
|
|
throw CloudASRError.emptyTranscript
|
|
}
|
|
return text
|
|
}
|
|
|
|
private static func parseZhipuText(from data: Data) -> String? {
|
|
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
|
return nil
|
|
}
|
|
return json["text"] as? String
|
|
}
|
|
|
|
fileprivate static func validateHTTP(response: URLResponse, data: Data) throws {
|
|
guard let http = response as? HTTPURLResponse else {
|
|
throw CloudASRError.transport("non-HTTP response")
|
|
}
|
|
guard (200..<300).contains(http.statusCode) else {
|
|
let message = parseErrorMessage(from: data)
|
|
throw CloudASRError.http(status: http.statusCode, message: message)
|
|
}
|
|
}
|
|
|
|
fileprivate static func parseErrorMessage(from data: Data) -> String? {
|
|
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
|
return nil
|
|
}
|
|
if let message = json["message"] as? String { return message }
|
|
if let error = json["error"] as? [String: Any],
|
|
let message = error["message"] as? String {
|
|
return message
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// MARK: - Alibaba Fun-ASR Flash (vocabulary_id + context)
|
|
|
|
/// `UserDefaults` is not `Sendable`; we only touch `persistence` on the
|
|
/// actor-isolated cloud ASR path, same as the previous `AppGroupStore` holder.
|
|
struct AlibabaFunASRClient: CloudASRTranscribing, @unchecked Sendable {
|
|
let apiKey: String
|
|
let model: String
|
|
let persistence: UserDefaults
|
|
let session: URLSession
|
|
|
|
func prepare(dictionary: PersonalDictionary) async throws {
|
|
_ = try await AlibabaVocabularySync.ensureVocabularyID(
|
|
dictionary: dictionary,
|
|
apiKey: apiKey,
|
|
targetModel: CloudASRModelCatalog.alibabaVocabularyTargetModel,
|
|
defaults: persistence,
|
|
session: session
|
|
)
|
|
}
|
|
|
|
func transcribe(
|
|
samples: [Float],
|
|
sampleRate: Int,
|
|
locale: Locale,
|
|
dictionary: PersonalDictionary
|
|
) async throws -> String {
|
|
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
|
|
|
let vocabularyID = try await AlibabaVocabularySync.ensureVocabularyID(
|
|
dictionary: dictionary,
|
|
apiKey: apiKey,
|
|
targetModel: CloudASRModelCatalog.alibabaVocabularyTargetModel,
|
|
defaults: persistence,
|
|
session: session
|
|
)
|
|
|
|
let dataURI = PCMSampleWavEncoder.dataURI(samples: samples, sampleRate: sampleRate)
|
|
let urlString = CloudASRModelCatalog.alibabaAPIBase + CloudASRModelCatalog.alibabaMultimodalPath
|
|
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
|
|
|
var messages: [[String: Any]] = []
|
|
let context = dictionary.alibabaContextText()
|
|
if !context.isEmpty {
|
|
messages.append([
|
|
"role": "user",
|
|
"content": [
|
|
["type": "input_text", "text": context],
|
|
],
|
|
])
|
|
}
|
|
messages.append([
|
|
"role": "user",
|
|
"content": [
|
|
[
|
|
"type": "input_audio",
|
|
"input_audio": ["data": dataURI],
|
|
],
|
|
],
|
|
])
|
|
|
|
var parameters: [String: Any] = [
|
|
"format": "wav",
|
|
"sample_rate": "\(sampleRate)",
|
|
]
|
|
if let vocabularyID, !vocabularyID.isEmpty {
|
|
parameters["vocabulary_id"] = vocabularyID
|
|
}
|
|
|
|
let body: [String: Any] = [
|
|
"model": model,
|
|
"input": ["messages": messages],
|
|
"parameters": parameters,
|
|
]
|
|
|
|
var request = URLRequest(url: url)
|
|
request.httpMethod = "POST"
|
|
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
|
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
|
request.setValue("disable", forHTTPHeaderField: "X-DashScope-SSE")
|
|
request.httpBody = try JSONSerialization.data(withJSONObject: body)
|
|
request.timeoutInterval = 90
|
|
|
|
let (data, response) = try await session.data(for: request)
|
|
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
|
guard let text = Self.parseText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
|
!text.isEmpty else {
|
|
throw CloudASRError.emptyTranscript
|
|
}
|
|
return text
|
|
}
|
|
|
|
private static func parseText(from data: Data) -> String? {
|
|
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
|
let output = json["output"] as? [String: Any] else {
|
|
return nil
|
|
}
|
|
if let text = output["text"] as? String { return text }
|
|
if let sentence = output["sentence"] as? [String: Any],
|
|
let text = sentence["text"] as? String {
|
|
return text
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// MARK: - Prompt-biased transcription (OpenAI / MiMo / custom)
|
|
|
|
struct PromptCloudASRClient: CloudASRTranscribing {
|
|
let providerId: String
|
|
let baseURL: String
|
|
let apiKey: String
|
|
let model: String
|
|
let session: URLSession
|
|
|
|
func prepare(dictionary: PersonalDictionary) async throws {}
|
|
|
|
func transcribe(
|
|
samples: [Float],
|
|
sampleRate: Int,
|
|
locale: Locale,
|
|
dictionary: PersonalDictionary
|
|
) async throws -> String {
|
|
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
|
if providerId == "mimo" {
|
|
return try await transcribeMiMo(
|
|
samples: samples,
|
|
sampleRate: sampleRate,
|
|
dictionary: dictionary
|
|
)
|
|
}
|
|
return try await transcribeOpenAIStyle(
|
|
samples: samples,
|
|
sampleRate: sampleRate,
|
|
dictionary: dictionary
|
|
)
|
|
}
|
|
|
|
private func transcribeOpenAIStyle(
|
|
samples: [Float],
|
|
sampleRate: Int,
|
|
dictionary: PersonalDictionary
|
|
) async throws -> String {
|
|
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
|
|
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
|
|
let urlString = "\(trimmedBase)/audio/transcriptions"
|
|
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
|
|
|
let boundary = "Boundary-\(UUID().uuidString)"
|
|
var body = Data()
|
|
func appendField(_ name: String, _ value: String) {
|
|
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
|
body.append("Content-Disposition: form-data; name=\"\(name)\"\r\n\r\n".data(using: .utf8)!)
|
|
body.append("\(value)\r\n".data(using: .utf8)!)
|
|
}
|
|
|
|
appendField("model", model)
|
|
let prompt = dictionary.asrPromptBias(maxCharacters: 600)
|
|
if !prompt.isEmpty {
|
|
appendField("prompt", prompt)
|
|
}
|
|
|
|
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
|
body.append("Content-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\n".data(using: .utf8)!)
|
|
body.append("Content-Type: audio/wav\r\n\r\n".data(using: .utf8)!)
|
|
body.append(wav)
|
|
body.append("\r\n".data(using: .utf8)!)
|
|
body.append("--\(boundary)--\r\n".data(using: .utf8)!)
|
|
|
|
var request = URLRequest(url: url)
|
|
request.httpMethod = "POST"
|
|
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
|
request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")
|
|
request.httpBody = body
|
|
request.timeoutInterval = 90
|
|
|
|
let (data, response) = try await session.data(for: request)
|
|
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
|
guard let text = Self.parseOpenAIText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
|
!text.isEmpty else {
|
|
throw CloudASRError.emptyTranscript
|
|
}
|
|
return text
|
|
}
|
|
|
|
private func transcribeMiMo(
|
|
samples: [Float],
|
|
sampleRate: Int,
|
|
dictionary: PersonalDictionary
|
|
) async throws -> String {
|
|
let dataURI = PCMSampleWavEncoder.dataURI(samples: samples, sampleRate: sampleRate)
|
|
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
|
|
let urlString = "\(trimmedBase)/chat/completions"
|
|
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
|
|
|
var userContent: [[String: Any]] = []
|
|
let prompt = dictionary.asrPromptBias()
|
|
if !prompt.isEmpty {
|
|
userContent.append(["type": "text", "text": prompt])
|
|
}
|
|
userContent.append([
|
|
"type": "input_audio",
|
|
"input_audio": ["data": dataURI],
|
|
])
|
|
|
|
let body: [String: Any] = [
|
|
"model": model,
|
|
"messages": [
|
|
["role": "user", "content": userContent],
|
|
],
|
|
"asr_options": ["language": "auto"],
|
|
]
|
|
|
|
var request = URLRequest(url: url)
|
|
request.httpMethod = "POST"
|
|
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
|
request.setValue(apiKey, forHTTPHeaderField: "api-key")
|
|
request.httpBody = try JSONSerialization.data(withJSONObject: body)
|
|
request.timeoutInterval = 90
|
|
|
|
let (data, response) = try await session.data(for: request)
|
|
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
|
guard let text = Self.parseChatCompletionText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
|
!text.isEmpty else {
|
|
throw CloudASRError.emptyTranscript
|
|
}
|
|
return text
|
|
}
|
|
|
|
private static func parseOpenAIText(from data: Data) -> String? {
|
|
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
|
return nil
|
|
}
|
|
return json["text"] as? String
|
|
}
|
|
|
|
private static func parseChatCompletionText(from data: Data) -> String? {
|
|
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
|
let choices = json["choices"] as? [[String: Any]],
|
|
let first = choices.first,
|
|
let message = first["message"] as? [String: Any] else {
|
|
return nil
|
|
}
|
|
return message["content"] as? String
|
|
}
|
|
}
|
|
|
|
// MARK: - Unsupported hosted ASR (Moonshot)
|
|
|
|
struct UnsupportedCloudASRClient: CloudASRTranscribing {
|
|
let providerId: String
|
|
|
|
func prepare(dictionary: PersonalDictionary) async throws {}
|
|
|
|
func transcribe(
|
|
samples: [Float],
|
|
sampleRate: Int,
|
|
locale: Locale,
|
|
dictionary: PersonalDictionary
|
|
) async throws -> String {
|
|
throw CloudASRError.providerUnsupported
|
|
}
|
|
}
|