feat(dictionary): add iCloud KVS sync, cloud ASR, and lexicon expansion
Mirror the personal dictionary through iCloud Key-Value Store with deterministic merge rules, main-app-only sync UI, and App Group as the keyboard runtime cache. Add cloud-engine ASR with dictionary bias and expand the bundled custom language model lexicon.
This commit is contained in:
@@ -121,9 +121,13 @@ public enum ASREvent: Sendable, Equatable {
|
||||
// MARK: - Factory
|
||||
|
||||
public enum ASRServiceFactory {
|
||||
/// Returns the on-device `SpeechAnalyzer` + `DictationTranscriber` backend.
|
||||
public static func make() -> ASRService {
|
||||
SpeechAnalyzerASR()
|
||||
/// Returns on-device SpeechAnalyzer for `local`, or the user's cloud
|
||||
/// ASR provider when `engineMode == "cloud"`.
|
||||
public static func make(store: AppGroupStore = AppGroupStore()) -> ASRService {
|
||||
if store.engineMode == "cloud" {
|
||||
return CloudASRService(store: store)
|
||||
}
|
||||
return SpeechAnalyzerASR()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -155,6 +155,16 @@ public struct AppGroupStore: @unchecked Sendable {
|
||||
|
||||
public func setPersonalDictionary(_ dictionary: PersonalDictionary) {
|
||||
mutateConfiguration { $0.personalDictionary = dictionary }
|
||||
AppGroupConfigDarwin.postConfigChanged()
|
||||
}
|
||||
|
||||
public var personalDictionaryICloudSyncEnabled: Bool {
|
||||
get { configuration.personalDictionaryICloudSyncEnabled }
|
||||
set { setPersonalDictionaryICloudSyncEnabled(newValue) }
|
||||
}
|
||||
|
||||
public func setPersonalDictionaryICloudSyncEnabled(_ enabled: Bool) {
|
||||
mutateConfiguration { $0.personalDictionaryICloudSyncEnabled = enabled }
|
||||
}
|
||||
|
||||
// MARK: - Client
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
// AlibabaVocabularySync.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Syncs PersonalDictionary → DashScope custom vocabulary (Fun-ASR Flash).
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum AlibabaVocabularySync {
|
||||
public enum Keys {
|
||||
public static let vocabularyId = "config.alibabaASRVocabularyId"
|
||||
public static let fingerprint = "config.alibabaASRVocabularyFingerprint"
|
||||
}
|
||||
|
||||
private static let vocabularyPrefix = "osgkb"
|
||||
|
||||
/// Returns a ready `vocabulary_id`, creating or updating the remote list as needed.
|
||||
public static func ensureVocabularyID(
|
||||
dictionary: PersonalDictionary,
|
||||
apiKey: String,
|
||||
targetModel: String = CloudASRModelCatalog.alibabaVocabularyTargetModel,
|
||||
defaults: UserDefaults,
|
||||
session: URLSession = .shared
|
||||
) async throws -> String? {
|
||||
let entries = dictionary.alibabaHotwordEntries()
|
||||
guard !entries.isEmpty else {
|
||||
clearCache(defaults: defaults)
|
||||
return nil
|
||||
}
|
||||
|
||||
let fingerprint = dictionary.vocabularySyncFingerprint()
|
||||
if let cachedID = defaults.string(forKey: Keys.vocabularyId),
|
||||
defaults.string(forKey: Keys.fingerprint) == fingerprint,
|
||||
!cachedID.isEmpty {
|
||||
return cachedID
|
||||
}
|
||||
|
||||
let url = try customizationURL()
|
||||
if let existingID = defaults.string(forKey: Keys.vocabularyId), !existingID.isEmpty {
|
||||
try await updateVocabulary(
|
||||
id: existingID,
|
||||
entries: entries,
|
||||
apiKey: apiKey,
|
||||
url: url,
|
||||
session: session
|
||||
)
|
||||
cache(id: existingID, fingerprint: fingerprint, defaults: defaults)
|
||||
return existingID
|
||||
}
|
||||
|
||||
let createdID = try await createVocabulary(
|
||||
entries: entries,
|
||||
targetModel: targetModel,
|
||||
apiKey: apiKey,
|
||||
url: url,
|
||||
session: session
|
||||
)
|
||||
cache(id: createdID, fingerprint: fingerprint, defaults: defaults)
|
||||
return createdID
|
||||
}
|
||||
|
||||
public static func clearCache(defaults: UserDefaults) {
|
||||
defaults.removeObject(forKey: Keys.vocabularyId)
|
||||
defaults.removeObject(forKey: Keys.fingerprint)
|
||||
}
|
||||
|
||||
private static func cache(id: String, fingerprint: String, defaults: UserDefaults) {
|
||||
defaults.set(id, forKey: Keys.vocabularyId)
|
||||
defaults.set(fingerprint, forKey: Keys.fingerprint)
|
||||
}
|
||||
|
||||
private static func customizationURL() throws -> URL {
|
||||
let raw = CloudASRModelCatalog.alibabaAPIBase + CloudASRModelCatalog.alibabaCustomizationPath
|
||||
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
|
||||
return url
|
||||
}
|
||||
|
||||
private static func createVocabulary(
|
||||
entries: [AlibabaHotwordEntry],
|
||||
targetModel: String,
|
||||
apiKey: String,
|
||||
url: URL,
|
||||
session: URLSession
|
||||
) async throws -> String {
|
||||
let vocabulary = entries.map { entry -> [String: Any] in
|
||||
var item: [String: Any] = ["text": entry.text, "weight": entry.weight]
|
||||
if let lang = entry.lang { item["lang"] = lang }
|
||||
return item
|
||||
}
|
||||
let body: [String: Any] = [
|
||||
"model": "speech-biasing",
|
||||
"input": [
|
||||
"action": "create_vocabulary",
|
||||
"target_model": targetModel,
|
||||
"prefix": vocabularyPrefix,
|
||||
"vocabulary": vocabulary,
|
||||
] as [String: Any],
|
||||
]
|
||||
let data = try await postJSON(body, to: url, apiKey: apiKey, session: session)
|
||||
guard let id = parseVocabularyID(from: data) else {
|
||||
throw CloudASRError.decoding("missing vocabulary_id")
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
private static func updateVocabulary(
|
||||
id: String,
|
||||
entries: [AlibabaHotwordEntry],
|
||||
apiKey: String,
|
||||
url: URL,
|
||||
session: URLSession
|
||||
) async throws {
|
||||
let vocabulary = entries.map { entry -> [String: Any] in
|
||||
var item: [String: Any] = ["text": entry.text, "weight": entry.weight]
|
||||
if let lang = entry.lang { item["lang"] = lang }
|
||||
return item
|
||||
}
|
||||
let body: [String: Any] = [
|
||||
"model": "speech-biasing",
|
||||
"input": [
|
||||
"action": "update_vocabulary",
|
||||
"vocabulary_id": id,
|
||||
"vocabulary": vocabulary,
|
||||
] as [String: Any],
|
||||
]
|
||||
_ = try await postJSON(body, to: url, apiKey: apiKey, session: session)
|
||||
}
|
||||
|
||||
private static func postJSON(
|
||||
_ body: [String: Any],
|
||||
to url: URL,
|
||||
apiKey: String,
|
||||
session: URLSession
|
||||
) async throws -> [String: Any] {
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
||||
request.httpBody = try JSONSerialization.data(withJSONObject: body)
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
guard let http = response as? HTTPURLResponse else {
|
||||
throw CloudASRError.transport("non-HTTP response")
|
||||
}
|
||||
let json = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any]
|
||||
guard (200..<300).contains(http.statusCode) else {
|
||||
let message = parseAPIErrorMessage(from: json)
|
||||
throw CloudASRError.http(status: http.statusCode, message: message)
|
||||
}
|
||||
return json ?? [:]
|
||||
}
|
||||
|
||||
private static func parseVocabularyID(from json: [String: Any]) -> String? {
|
||||
if let output = json["output"] as? [String: Any],
|
||||
let id = output["vocabulary_id"] as? String {
|
||||
return id
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
private static func parseAPIErrorMessage(from json: [String: Any]?) -> String? {
|
||||
guard let json else { return nil }
|
||||
if let message = json["message"] as? String { return message }
|
||||
if let error = json["error"] as? [String: Any],
|
||||
let message = error["message"] as? String {
|
||||
return message
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,411 @@
|
||||
// CloudASRClients.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Provider-specific cloud ASR backends with personal-dictionary bias.
|
||||
|
||||
import Foundation
|
||||
|
||||
public protocol CloudASRTranscribing: Sendable {
|
||||
func prepare(dictionary: PersonalDictionary) async throws
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String
|
||||
}
|
||||
|
||||
public enum CloudASRClientFactory {
|
||||
public static func make(store: AppGroupStore, session: URLSession = .shared) -> CloudASRTranscribing {
|
||||
let strategy = CloudASRModelCatalog.strategy(for: store.providerId)
|
||||
switch strategy {
|
||||
case .zhipuHotwords:
|
||||
return ZhipuCloudASRClient(
|
||||
apiKey: store.apiKey,
|
||||
model: CloudASRModelCatalog.defaultModel(for: store.providerId),
|
||||
session: session
|
||||
)
|
||||
case .alibabaVocabulary:
|
||||
return AlibabaFunASRClient(
|
||||
apiKey: store.apiKey,
|
||||
model: CloudASRModelCatalog.defaultModel(for: store.providerId),
|
||||
store: store,
|
||||
session: session
|
||||
)
|
||||
case .prompt:
|
||||
return PromptCloudASRClient(
|
||||
providerId: store.providerId,
|
||||
baseURL: store.baseURL,
|
||||
apiKey: store.apiKey,
|
||||
model: CloudASRModelCatalog.defaultModel(for: store.providerId),
|
||||
session: session
|
||||
)
|
||||
case .localFallback:
|
||||
return UnsupportedCloudASRClient(providerId: store.providerId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Zhipu (hotwords + prompt)
|
||||
|
||||
struct ZhipuCloudASRClient: CloudASRTranscribing {
|
||||
let apiKey: String
|
||||
let model: String
|
||||
let session: URLSession
|
||||
|
||||
private static let maxDurationSeconds: TimeInterval = 30
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
let duration = Double(samples.count) / Double(sampleRate)
|
||||
guard duration <= Self.maxDurationSeconds else { throw CloudASRError.audioTooLong }
|
||||
|
||||
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
|
||||
let urlString = "https://open.bigmodel.cn/api/paas/v4\(CloudASRModelCatalog.zhipuTranscriptionPath)"
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
|
||||
let boundary = "Boundary-\(UUID().uuidString)"
|
||||
var body = Data()
|
||||
func appendField(_ name: String, _ value: String) {
|
||||
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Disposition: form-data; name=\"\(name)\"\r\n\r\n".data(using: .utf8)!)
|
||||
body.append("\(value)\r\n".data(using: .utf8)!)
|
||||
}
|
||||
|
||||
appendField("model", model)
|
||||
appendField("stream", "false")
|
||||
|
||||
let hotwords = dictionary.asrHotwords()
|
||||
if !hotwords.isEmpty,
|
||||
let hotwordsJSON = try? JSONSerialization.data(withJSONObject: hotwords),
|
||||
let hotwordsString = String(data: hotwordsJSON, encoding: .utf8) {
|
||||
appendField("hotwords", hotwordsString)
|
||||
}
|
||||
|
||||
let prompt = dictionary.asrPromptBias()
|
||||
if !prompt.isEmpty {
|
||||
appendField("prompt", prompt)
|
||||
}
|
||||
|
||||
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Type: audio/wav\r\n\r\n".data(using: .utf8)!)
|
||||
body.append(wav)
|
||||
body.append("\r\n".data(using: .utf8)!)
|
||||
body.append("--\(boundary)--\r\n".data(using: .utf8)!)
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
||||
request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")
|
||||
request.httpBody = body
|
||||
request.timeoutInterval = 60
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
try Self.validateHTTP(response: response, data: data)
|
||||
guard let text = Self.parseZhipuText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty else {
|
||||
throw CloudASRError.emptyTranscript
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
private static func parseZhipuText(from data: Data) -> String? {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
||||
return nil
|
||||
}
|
||||
return json["text"] as? String
|
||||
}
|
||||
|
||||
fileprivate static func validateHTTP(response: URLResponse, data: Data) throws {
|
||||
guard let http = response as? HTTPURLResponse else {
|
||||
throw CloudASRError.transport("non-HTTP response")
|
||||
}
|
||||
guard (200..<300).contains(http.statusCode) else {
|
||||
let message = parseErrorMessage(from: data)
|
||||
throw CloudASRError.http(status: http.statusCode, message: message)
|
||||
}
|
||||
}
|
||||
|
||||
fileprivate static func parseErrorMessage(from data: Data) -> String? {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
||||
return nil
|
||||
}
|
||||
if let message = json["message"] as? String { return message }
|
||||
if let error = json["error"] as? [String: Any],
|
||||
let message = error["message"] as? String {
|
||||
return message
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Alibaba Fun-ASR Flash (vocabulary_id + context)
|
||||
|
||||
struct AlibabaFunASRClient: CloudASRTranscribing {
|
||||
let apiKey: String
|
||||
let model: String
|
||||
// Hold the (@unchecked Sendable) AppGroupStore rather than a raw
|
||||
// UserDefaults so this struct stays Sendable under strict concurrency.
|
||||
let store: AppGroupStore
|
||||
let session: URLSession
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {
|
||||
_ = try await AlibabaVocabularySync.ensureVocabularyID(
|
||||
dictionary: dictionary,
|
||||
apiKey: apiKey,
|
||||
targetModel: CloudASRModelCatalog.alibabaVocabularyTargetModel,
|
||||
defaults: store.defaults,
|
||||
session: session
|
||||
)
|
||||
}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
|
||||
let vocabularyID = try await AlibabaVocabularySync.ensureVocabularyID(
|
||||
dictionary: dictionary,
|
||||
apiKey: apiKey,
|
||||
targetModel: CloudASRModelCatalog.alibabaVocabularyTargetModel,
|
||||
defaults: store.defaults,
|
||||
session: session
|
||||
)
|
||||
|
||||
let dataURI = PCMSampleWavEncoder.dataURI(samples: samples, sampleRate: sampleRate)
|
||||
let urlString = CloudASRModelCatalog.alibabaAPIBase + CloudASRModelCatalog.alibabaMultimodalPath
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
|
||||
var messages: [[String: Any]] = []
|
||||
let context = dictionary.alibabaContextText()
|
||||
if !context.isEmpty {
|
||||
messages.append([
|
||||
"role": "user",
|
||||
"content": [
|
||||
["type": "input_text", "text": context],
|
||||
],
|
||||
])
|
||||
}
|
||||
messages.append([
|
||||
"role": "user",
|
||||
"content": [
|
||||
[
|
||||
"type": "input_audio",
|
||||
"input_audio": ["data": dataURI],
|
||||
],
|
||||
],
|
||||
])
|
||||
|
||||
var parameters: [String: Any] = [
|
||||
"format": "wav",
|
||||
"sample_rate": "\(sampleRate)",
|
||||
]
|
||||
if let vocabularyID, !vocabularyID.isEmpty {
|
||||
parameters["vocabulary_id"] = vocabularyID
|
||||
}
|
||||
|
||||
let body: [String: Any] = [
|
||||
"model": model,
|
||||
"input": ["messages": messages],
|
||||
"parameters": parameters,
|
||||
]
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
||||
request.setValue("disable", forHTTPHeaderField: "X-DashScope-SSE")
|
||||
request.httpBody = try JSONSerialization.data(withJSONObject: body)
|
||||
request.timeoutInterval = 90
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
||||
guard let text = Self.parseText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty else {
|
||||
throw CloudASRError.emptyTranscript
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
private static func parseText(from data: Data) -> String? {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||
let output = json["output"] as? [String: Any] else {
|
||||
return nil
|
||||
}
|
||||
if let text = output["text"] as? String { return text }
|
||||
if let sentence = output["sentence"] as? [String: Any],
|
||||
let text = sentence["text"] as? String {
|
||||
return text
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Prompt-biased transcription (OpenAI / MiMo / custom)
|
||||
|
||||
struct PromptCloudASRClient: CloudASRTranscribing {
|
||||
let providerId: String
|
||||
let baseURL: String
|
||||
let apiKey: String
|
||||
let model: String
|
||||
let session: URLSession
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
|
||||
if providerId == "mimo" {
|
||||
return try await transcribeMiMo(
|
||||
samples: samples,
|
||||
sampleRate: sampleRate,
|
||||
dictionary: dictionary
|
||||
)
|
||||
}
|
||||
return try await transcribeOpenAIStyle(
|
||||
samples: samples,
|
||||
sampleRate: sampleRate,
|
||||
dictionary: dictionary
|
||||
)
|
||||
}
|
||||
|
||||
private func transcribeOpenAIStyle(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
|
||||
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
|
||||
let urlString = "\(trimmedBase)/audio/transcriptions"
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
|
||||
let boundary = "Boundary-\(UUID().uuidString)"
|
||||
var body = Data()
|
||||
func appendField(_ name: String, _ value: String) {
|
||||
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Disposition: form-data; name=\"\(name)\"\r\n\r\n".data(using: .utf8)!)
|
||||
body.append("\(value)\r\n".data(using: .utf8)!)
|
||||
}
|
||||
|
||||
appendField("model", model)
|
||||
let prompt = dictionary.asrPromptBias(maxCharacters: 600)
|
||||
if !prompt.isEmpty {
|
||||
appendField("prompt", prompt)
|
||||
}
|
||||
|
||||
body.append("--\(boundary)\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\n".data(using: .utf8)!)
|
||||
body.append("Content-Type: audio/wav\r\n\r\n".data(using: .utf8)!)
|
||||
body.append(wav)
|
||||
body.append("\r\n".data(using: .utf8)!)
|
||||
body.append("--\(boundary)--\r\n".data(using: .utf8)!)
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
|
||||
request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")
|
||||
request.httpBody = body
|
||||
request.timeoutInterval = 90
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
||||
guard let text = Self.parseOpenAIText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty else {
|
||||
throw CloudASRError.emptyTranscript
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
private func transcribeMiMo(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
let dataURI = PCMSampleWavEncoder.dataURI(samples: samples, sampleRate: sampleRate)
|
||||
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
|
||||
let urlString = "\(trimmedBase)/chat/completions"
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
|
||||
var userContent: [[String: Any]] = []
|
||||
let prompt = dictionary.asrPromptBias()
|
||||
if !prompt.isEmpty {
|
||||
userContent.append(["type": "text", "text": prompt])
|
||||
}
|
||||
userContent.append([
|
||||
"type": "input_audio",
|
||||
"input_audio": ["data": dataURI],
|
||||
])
|
||||
|
||||
let body: [String: Any] = [
|
||||
"model": model,
|
||||
"messages": [
|
||||
["role": "user", "content": userContent],
|
||||
],
|
||||
"asr_options": ["language": "auto"],
|
||||
]
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
request.setValue(apiKey, forHTTPHeaderField: "api-key")
|
||||
request.httpBody = try JSONSerialization.data(withJSONObject: body)
|
||||
request.timeoutInterval = 90
|
||||
|
||||
let (data, response) = try await session.data(for: request)
|
||||
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
|
||||
guard let text = Self.parseChatCompletionText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty else {
|
||||
throw CloudASRError.emptyTranscript
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
private static func parseOpenAIText(from data: Data) -> String? {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
||||
return nil
|
||||
}
|
||||
return json["text"] as? String
|
||||
}
|
||||
|
||||
private static func parseChatCompletionText(from data: Data) -> String? {
|
||||
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||
let choices = json["choices"] as? [[String: Any]],
|
||||
let first = choices.first,
|
||||
let message = first["message"] as? [String: Any] else {
|
||||
return nil
|
||||
}
|
||||
return message["content"] as? String
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Unsupported hosted ASR (Moonshot)
|
||||
|
||||
struct UnsupportedCloudASRClient: CloudASRTranscribing {
|
||||
let providerId: String
|
||||
|
||||
func prepare(dictionary: PersonalDictionary) async throws {}
|
||||
|
||||
func transcribe(
|
||||
samples: [Float],
|
||||
sampleRate: Int,
|
||||
locale: Locale,
|
||||
dictionary: PersonalDictionary
|
||||
) async throws -> String {
|
||||
throw CloudASRError.providerUnsupported
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
// CloudASRService.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Cloud-engine ASR: uploads PCM chunks to the user's configured provider
|
||||
// with personal-dictionary bias. Moonshot falls back to on-device ASR.
|
||||
|
||||
import Foundation
|
||||
import os
|
||||
|
||||
public final class CloudASRService: ASRService, @unchecked Sendable {
|
||||
private let store: AppGroupStore
|
||||
private let session: URLSession
|
||||
private let localFallback: ASRService
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
private var client: CloudASRTranscribing?
|
||||
private var usesLocalFallback = false
|
||||
private var boundProviderId: String?
|
||||
private var cancelled = false
|
||||
|
||||
public init(
|
||||
store: AppGroupStore = AppGroupStore(),
|
||||
session: URLSession = .shared,
|
||||
localFallback: ASRService? = nil
|
||||
) {
|
||||
self.store = store
|
||||
self.session = session
|
||||
// `SpeechAnalyzerASR` is internal, so it can't appear in a public
|
||||
// default argument value — resolve the fallback in the body instead.
|
||||
self.localFallback = localFallback ?? SpeechAnalyzerASR()
|
||||
}
|
||||
|
||||
public func resetForNewUtterance() {
|
||||
lock.withLock { cancelled = false }
|
||||
if usesLocalFallback {
|
||||
localFallback.resetForNewUtterance()
|
||||
}
|
||||
}
|
||||
|
||||
public func warmup(locale: Locale) async {
|
||||
bindClientIfNeeded()
|
||||
if usesLocalFallback {
|
||||
await localFallback.warmup(locale: locale)
|
||||
return
|
||||
}
|
||||
guard let client = lock.withLock({ client }) else { return }
|
||||
do {
|
||||
try await client.prepare(dictionary: store.personalDictionary)
|
||||
} catch {
|
||||
OSGLog.asr.warning("cloud ASR vocabulary prepare failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
|
||||
guard !samples.isEmpty else { return .success("") }
|
||||
if Task.isCancelled || lock.withLock({ cancelled }) { return .cancelled }
|
||||
|
||||
bindClientIfNeeded()
|
||||
if usesLocalFallback {
|
||||
return await localFallback.transcribeChunk(samples: samples, locale: locale)
|
||||
}
|
||||
|
||||
guard let client = lock.withLock({ client }) else {
|
||||
return .failure(CloudASRError.providerUnsupported.localizedDescription)
|
||||
}
|
||||
|
||||
do {
|
||||
let text = try await client.transcribe(
|
||||
samples: samples,
|
||||
sampleRate: 16_000,
|
||||
locale: locale,
|
||||
dictionary: store.personalDictionary
|
||||
)
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return trimmed.isEmpty ? .success("") : .success(trimmed)
|
||||
} catch is CancellationError {
|
||||
return .cancelled
|
||||
} catch {
|
||||
return .failure(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
public func transcribe(
|
||||
stream: AsyncStream<AudioBufferSnapshot>,
|
||||
locale: Locale
|
||||
) -> AsyncStream<ASREvent> {
|
||||
bindClientIfNeeded()
|
||||
if usesLocalFallback {
|
||||
return localFallback.transcribe(stream: stream, locale: locale)
|
||||
}
|
||||
|
||||
return AsyncStream { continuation in
|
||||
continuation.yield(.capability(onDeviceSupported: false))
|
||||
let task = Task {
|
||||
var samples: [Float] = []
|
||||
for await snap in stream {
|
||||
if Task.isCancelled { break }
|
||||
samples.append(contentsOf: snap.samples)
|
||||
}
|
||||
guard !Task.isCancelled, !self.lock.withLock({ self.cancelled }) else {
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
guard !samples.isEmpty else {
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
|
||||
switch await self.transcribeChunk(samples: samples, locale: locale) {
|
||||
case .success(let text):
|
||||
if text.isEmpty {
|
||||
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
|
||||
} else {
|
||||
continuation.yield(.final(text))
|
||||
}
|
||||
case .failure(let message):
|
||||
continuation.yield(.error(message))
|
||||
case .cancelled:
|
||||
break
|
||||
}
|
||||
continuation.finish()
|
||||
}
|
||||
continuation.onTermination = { @Sendable _ in
|
||||
task.cancel()
|
||||
self.cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public func cancel() {
|
||||
lock.withLock { cancelled = true }
|
||||
localFallback.cancel()
|
||||
}
|
||||
|
||||
private func bindClientIfNeeded() {
|
||||
let providerId = store.providerId
|
||||
let strategy = CloudASRModelCatalog.strategy(for: providerId)
|
||||
lock.withLock {
|
||||
guard boundProviderId != providerId else { return }
|
||||
boundProviderId = providerId
|
||||
usesLocalFallback = strategy == .localFallback
|
||||
client = usesLocalFallback
|
||||
? nil
|
||||
: CloudASRClientFactory.make(store: store, session: session)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -115,7 +115,7 @@ public final class LiveDictationController: ObservableObject {
|
||||
private var didInstallTap = false
|
||||
|
||||
public init(asr: ASRService? = nil) {
|
||||
self.asr = asr ?? ASRServiceFactory.make()
|
||||
self.asr = asr ?? ASRServiceFactory.make(store: AppGroupStore())
|
||||
}
|
||||
|
||||
/// Start dictation using a persisted settings locale id (`auto`, `zh-Hans`, …).
|
||||
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
// PersonalDictionaryCloudSync.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Mirrors the personal dictionary through iCloud Key-Value Store while
|
||||
// keeping App Group UserDefaults as the keyboard extension's runtime
|
||||
// source of truth. Intended for main-app call sites only.
|
||||
|
||||
import Foundation
|
||||
|
||||
public extension Notification.Name {
|
||||
/// Posted after a remote KVS pull updates the App Group dictionary.
|
||||
static let personalDictionaryDidSyncFromCloud = Notification.Name(
|
||||
"com.osgkeyboard.personalDictionary.didSyncFromCloud"
|
||||
)
|
||||
}
|
||||
|
||||
public enum PersonalDictionaryCloudSyncError: Error, Equatable, Sendable {
|
||||
case payloadTooLarge(byteCount: Int)
|
||||
case encodeFailed
|
||||
case decodeFailed
|
||||
}
|
||||
|
||||
@MainActor
|
||||
public final class PersonalDictionaryCloudSync {
|
||||
public static let shared = PersonalDictionaryCloudSync()
|
||||
|
||||
public static let kvsKey = "personalDictionary.v1"
|
||||
/// Stay below the ~1 MB per-key KVS limit.
|
||||
public static let maxPayloadBytes = 900_000
|
||||
|
||||
private let kvs: UbiquitousKeyValueStoreing
|
||||
private let makeStore: () -> AppGroupStore
|
||||
private var externalChangeObserver: NSObjectProtocol?
|
||||
|
||||
public init(
|
||||
kvs: UbiquitousKeyValueStoreing = NSUbiquitousKeyValueStore.default,
|
||||
makeStore: @escaping () -> AppGroupStore = { AppGroupStore() }
|
||||
) {
|
||||
self.kvs = kvs
|
||||
self.makeStore = makeStore
|
||||
}
|
||||
|
||||
// MARK: - Lifecycle
|
||||
|
||||
public func startObservingExternalChanges() {
|
||||
guard externalChangeObserver == nil else { return }
|
||||
externalChangeObserver = NotificationCenter.default.addObserver(
|
||||
forName: NSUbiquitousKeyValueStore.didChangeExternallyNotification,
|
||||
object: nil,
|
||||
queue: .main
|
||||
) { [weak self] _ in
|
||||
guard let self else { return }
|
||||
Task { @MainActor in
|
||||
await self.pullAndMergeIfEnabled()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public func stopObservingExternalChanges() {
|
||||
if let externalChangeObserver {
|
||||
NotificationCenter.default.removeObserver(externalChangeObserver)
|
||||
self.externalChangeObserver = nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Pull remote changes on launch / foreground when sync is enabled.
|
||||
public func pullAndMergeIfEnabled() async {
|
||||
let store = makeStore()
|
||||
guard store.personalDictionaryICloudSyncEnabled else { return }
|
||||
await pullAndMerge(store: store)
|
||||
}
|
||||
|
||||
/// Push the current local dictionary when sync is enabled.
|
||||
public func pushLocalIfEnabled(_ dictionary: PersonalDictionary) async throws {
|
||||
let store = makeStore()
|
||||
guard store.personalDictionaryICloudSyncEnabled else { return }
|
||||
try push(dictionary)
|
||||
}
|
||||
|
||||
/// Enable sync: merge local + remote, persist locally, then upload.
|
||||
public func enableSync() async throws {
|
||||
let store = makeStore()
|
||||
store.setPersonalDictionaryICloudSyncEnabled(true)
|
||||
|
||||
let local = store.personalDictionary
|
||||
let remote = loadRemote() ?? .empty
|
||||
let merged = PersonalDictionary.merge(local: local, remote: remote)
|
||||
store.setPersonalDictionary(merged)
|
||||
try push(merged)
|
||||
}
|
||||
|
||||
public func disableSync() {
|
||||
makeStore().setPersonalDictionaryICloudSyncEnabled(false)
|
||||
}
|
||||
|
||||
// MARK: - Core operations
|
||||
|
||||
public func pullAndMerge(store: AppGroupStore) async {
|
||||
guard store.personalDictionaryICloudSyncEnabled else { return }
|
||||
|
||||
let local = store.personalDictionary
|
||||
guard let remote = loadRemote() else { return }
|
||||
|
||||
let merged = PersonalDictionary.merge(local: local, remote: remote)
|
||||
guard merged != local else { return }
|
||||
|
||||
store.setPersonalDictionary(merged)
|
||||
NotificationCenter.default.post(name: .personalDictionaryDidSyncFromCloud, object: nil)
|
||||
}
|
||||
|
||||
public func push(_ dictionary: PersonalDictionary) throws {
|
||||
var payload = dictionary
|
||||
payload.lastSyncedAt = Date()
|
||||
let data = try encode(payload)
|
||||
kvs.set(data, forKey: Self.kvsKey)
|
||||
_ = kvs.synchronize()
|
||||
}
|
||||
|
||||
public func loadRemote() -> PersonalDictionary? {
|
||||
guard let data = kvs.data(forKey: Self.kvsKey) else { return nil }
|
||||
return try? decode(data)
|
||||
}
|
||||
|
||||
// MARK: - Encoding
|
||||
|
||||
public func encode(_ dictionary: PersonalDictionary) throws -> Data {
|
||||
let encoder = JSONEncoder()
|
||||
encoder.dateEncodingStrategy = .iso8601
|
||||
guard let data = try? encoder.encode(dictionary) else {
|
||||
throw PersonalDictionaryCloudSyncError.encodeFailed
|
||||
}
|
||||
guard data.count <= Self.maxPayloadBytes else {
|
||||
throw PersonalDictionaryCloudSyncError.payloadTooLarge(byteCount: data.count)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
public func decode(_ data: Data) throws -> PersonalDictionary {
|
||||
let decoder = JSONDecoder()
|
||||
decoder.dateDecodingStrategy = .iso8601
|
||||
guard let dictionary = try? decoder.decode(PersonalDictionary.self, from: data) else {
|
||||
throw PersonalDictionaryCloudSyncError.decodeFailed
|
||||
}
|
||||
return dictionary
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
// UbiquitousKeyValueStoreing.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Test seam around `NSUbiquitousKeyValueStore`.
|
||||
|
||||
import Foundation
|
||||
|
||||
public protocol UbiquitousKeyValueStoreing: AnyObject {
|
||||
func data(forKey key: String) -> Data?
|
||||
func set(_ value: Data?, forKey key: String)
|
||||
@discardableResult
|
||||
func synchronize() -> Bool
|
||||
}
|
||||
|
||||
extension NSUbiquitousKeyValueStore: UbiquitousKeyValueStoreing {}
|
||||
@@ -10,8 +10,8 @@
|
||||
// English dictation while halving the network round-trip.
|
||||
//
|
||||
// Engine matrix:
|
||||
// - `engineMode == "cloud"` → on-device ASR, then user's cloud LLM
|
||||
// - `engineMode == "local"` → on-device ASR, then built-in DeepSeek
|
||||
// - `engineMode == "cloud"` → provider cloud ASR + user's cloud LLM
|
||||
// - `engineMode == "local"` → on-device ASR + built-in DeepSeek
|
||||
// - Ultra-short, structure-free utterances skip the LLM entirely
|
||||
// - Cloud without API key → raw + `.missingAPIKey` warning
|
||||
// - Local without build key → raw + `.missingAPIKey` warning
|
||||
|
||||
Reference in New Issue
Block a user