feat(dictionary): add iCloud KVS sync, cloud ASR, and lexicon expansion

Mirror the personal dictionary through iCloud Key-Value Store with
deterministic merge rules, main-app-only sync UI, and App Group as the
keyboard runtime cache. Add cloud-engine ASR with dictionary bias and
expand the bundled custom language model lexicon.
This commit is contained in:
Rocky
2026-07-06 22:44:14 +08:00
parent 07df14b546
commit bf844caa7f
40 changed files with 5692 additions and 106 deletions
@@ -30,6 +30,8 @@ public struct AppGroupConfiguration: Sendable, Equatable {
public static let detectedAppContext = "config.detectedAppContext"
public static let detectedAppContextAt = "config.detectedAppContextAt"
public static let personalDictionary = "config.personalDictionary.v1"
/// When true, the main app mirrors the personal dictionary via iCloud KVS.
public static let personalDictionaryICloudSyncEnabled = "config.personalDictionary.iCloudSyncEnabled"
/// When true, the host app auto-returns to the source app after a cold-start handoff.
public static let flowSkipAppSwitch = "config.flowSkipAppSwitch"
/// Raw `FlowInactivityDuration` value; session expires after this idle window.
@@ -53,6 +55,8 @@ public struct AppGroupConfiguration: Sendable, Equatable {
public var cursorDragNavigationEnabled: Bool
public var polishIntensity: PolishIntensity
public var personalDictionary: PersonalDictionary
/// Opt-in iCloud KVS sync for the personal dictionary (main app only).
public var personalDictionaryICloudSyncEnabled: Bool
/// Auto-return to the host app after `startflow` cold start (default on).
public var flowSkipAppSwitch: Bool
/// Idle timeout before the Flow session ends; resets on each utterance.
@@ -154,6 +158,12 @@ public struct AppGroupConfiguration: Sendable, Equatable {
}(),
polishIntensity: resolvePolishIntensity(from: defaults),
personalDictionary: decodePersonalDictionary(from: defaults),
personalDictionaryICloudSyncEnabled: {
if defaults.object(forKey: Keys.personalDictionaryICloudSyncEnabled) == nil {
return true
}
return defaults.bool(forKey: Keys.personalDictionaryICloudSyncEnabled)
}(),
flowSkipAppSwitch: {
if defaults.object(forKey: Keys.flowSkipAppSwitch) == nil {
return true
@@ -212,6 +222,7 @@ public struct AppGroupConfiguration: Sendable, Equatable {
defaults.set(polishIntensity.rawValue, forKey: Keys.polishIntensity)
defaults.set(flowSkipAppSwitch, forKey: Keys.flowSkipAppSwitch)
defaults.set(flowInactivityDuration.rawValue, forKey: Keys.flowInactivityDuration)
defaults.set(personalDictionaryICloudSyncEnabled, forKey: Keys.personalDictionaryICloudSyncEnabled)
Self.encodePersonalDictionary(personalDictionary, to: defaults)
}
@@ -0,0 +1,121 @@
// CloudASRModels.swift
// OSGKeyboard · Shared
//
// Cloud-engine ASR routing: which provider uses official hotwords /
// vocabulary APIs vs. a transcription prompt bias.
import Foundation
/// How a cloud provider applies the user's personal dictionary during ASR.
public enum CloudASRStrategy: String, Sendable, Equatable {
/// GLM-ASR `hotwords` + optional `prompt`.
case zhipuHotwords
/// Fun-ASR managed `vocabulary_id` + context text.
case alibabaVocabulary
/// OpenAI / MiMo / `prompt` on transcription APIs.
case prompt
/// Moonshot API 退 ASR
case localFallback
}
public enum CloudASRError: Error, LocalizedError, Sendable, Equatable {
case noAPIKey
case invalidURL
case http(status: Int, message: String?)
case decoding(String)
case transport(String)
case emptyTranscript
case audioTooLong
case providerUnsupported
public var errorDescription: String? {
switch self {
case .noAPIKey:
return SharedL10n.string("error.cloudASR.noAPIKey")
case .invalidURL:
return SharedL10n.string("error.cloudASR.invalidURL")
case .http(let status, let message):
if let message, !message.isEmpty {
return SharedL10n.format("error.cloudASR.httpWithMessage", status, message)
}
return SharedL10n.format("error.cloudASR.http", status)
case .decoding(let detail):
return SharedL10n.format("error.cloudASR.decoding", detail)
case .transport(let detail):
return SharedL10n.format("error.cloudASR.transport", detail)
case .emptyTranscript:
return SharedL10n.string("error.cloudASR.emptyTranscript")
case .audioTooLong:
return SharedL10n.string("error.cloudASR.audioTooLong")
case .providerUnsupported:
return SharedL10n.string("error.cloudASR.providerUnsupported")
}
}
}
public enum CloudASRModelCatalog {
/// Sync Fun-ASR Flash base64 upload, 5 min, supports context + vocabulary.
public static let alibabaFunASRFlash = "fun-asr-flash-2026-06-15"
/// Must match the ASR model used at recognition time.
public static let alibabaVocabularyTargetModel = alibabaFunASRFlash
public static let zhipuGLMASR = "glm-asr-2512"
public static let openAITranscribe = "gpt-4o-mini-transcribe"
public static let openAIWhisper = "whisper-1"
public static let mimoASR = "mimo-v2.5-asr"
public static let alibabaAPIBase = "https://dashscope.aliyuncs.com/api/v1"
public static let alibabaCustomizationPath = "/services/audio/asr/customization"
public static let alibabaMultimodalPath = "/services/aigc/multimodal-generation/generation"
public static let zhipuTranscriptionPath = "/audio/transcriptions"
public static func strategy(for providerId: String) -> CloudASRStrategy {
switch providerId {
case "zhipu":
return .zhipuHotwords
case "qwen":
return .alibabaVocabulary
case "moonshot":
return .localFallback
case "openai", "mimo", "custom":
return .prompt
default:
return .prompt
}
}
public static func defaultModel(for providerId: String) -> String {
switch providerId {
case "zhipu":
return zhipuGLMASR
case "qwen":
return alibabaFunASRFlash
case "mimo":
return mimoASR
case "openai", "custom":
return openAITranscribe
default:
return openAITranscribe
}
}
}
extension LLMProvider {
public var cloudASRStrategy: CloudASRStrategy {
CloudASRModelCatalog.strategy(for: id)
}
public var defaultCloudASRModel: String {
CloudASRModelCatalog.defaultModel(for: id)
}
/// Official hotwords / vocabulary APIs during cloud ASR (not prompt-only bias).
public var supportsPersonalDictionaryCloudASR: Bool {
switch cloudASRStrategy {
case .zhipuHotwords, .alibabaVocabulary:
return true
case .prompt, .localFallback:
return false
}
}
}
@@ -0,0 +1,114 @@
// PersonalDictionary+ASRBias.swift
// OSGKeyboard · Shared
//
// Formats the user dictionary for cloud ASR bias (hotwords, Alibaba
// vocabulary entries, or Whisper-style prompt fragments).
import Foundation
import CryptoKit
public struct AlibabaHotwordEntry: Codable, Sendable, Equatable {
public let text: String
public let weight: Int
public let lang: String?
public init(text: String, weight: Int = 4, lang: String? = nil) {
self.text = text
self.weight = weight
self.lang = lang
}
}
extension PersonalDictionary {
/// Stable fingerprint used to decide when to refresh Alibaba vocabulary.
public func vocabularySyncFingerprint() -> String {
let payload = effectiveEntries
.sorted { $0.term.localizedCaseInsensitiveCompare($1.term) == .orderedAscending }
.map { entry in
let aliases = entry.aliases
.sorted { $0.localizedCaseInsensitiveCompare($1) == .orderedAscending }
.joined(separator: ",")
return "\(entry.term.lowercased())|\(aliases)"
}
.joined(separator: ";")
let digest = SHA256.hash(data: Data(payload.utf8))
return digest.map { String(format: "%02x", $0) }.joined()
}
/// `hotwords` canonical terms only (aliases go to `asrPromptBias`).
public func asrHotwords(maxCount: Int = 100) -> [String] {
var seen = Set<String>()
var words: [String] = []
for entry in effectiveEntries {
let term = entry.term.trimmingCharacters(in: .whitespacesAndNewlines)
guard !term.isEmpty else { continue }
let key = term.lowercased()
guard seen.insert(key).inserted else { continue }
words.append(term)
if words.count >= maxCount { break }
}
return words
}
/// `text` + `weight` (+ optional `lang`).
public func alibabaHotwordEntries(maxCount: Int = 500, defaultWeight: Int = 4) -> [AlibabaHotwordEntry] {
var seen = Set<String>()
var entries: [AlibabaHotwordEntry] = []
for entry in effectiveEntries {
let term = entry.term.trimmingCharacters(in: .whitespacesAndNewlines)
guard !term.isEmpty, term.count <= 15 else { continue }
let key = term.lowercased()
guard seen.insert(key).inserted else { continue }
entries.append(
AlibabaHotwordEntry(
text: term,
weight: defaultWeight,
lang: Self.inferAlibabaLang(for: term)
)
)
if entries.count >= maxCount { break }
}
return entries
}
/// Whisper / OpenAI-style short prompt bias (also used by MiMo text hint).
public func asrPromptBias(maxCharacters: Int = 800) -> String {
let entries = effectiveEntries
guard !entries.isEmpty else { return "" }
var lines: [String] = []
for entry in entries {
if entry.aliases.isEmpty {
lines.append(entry.term)
} else {
let aliasHint = entry.aliases.prefix(4).joined(separator: ", ")
lines.append("\(entry.term)(常见误识别:\(aliasHint)")
}
let joined = lines.joined(separator: "")
if joined.count > maxCharacters {
if lines.count == 1 {
return String(joined.prefix(maxCharacters))
}
lines.removeLast()
break
}
}
guard !lines.isEmpty else { return "" }
let body = lines.joined(separator: "")
return "用户专有词汇,转写时请优先使用以下标准写法:\(body)"
}
/// Compact domain context for Alibaba Fun-ASR Flash `input_text`.
public func alibabaContextText(maxCharacters: Int = 1200) -> String {
let prompt = asrPromptBias(maxCharacters: maxCharacters)
guard !prompt.isEmpty else { return "" }
return prompt
}
private static func inferAlibabaLang(for term: String) -> String? {
let hasNonASCII = term.unicodeScalars.contains { !$0.isASCII }
if hasNonASCII { return "zh" }
return "en"
}
}
@@ -0,0 +1,91 @@
// PersonalDictionary+Merging.swift
// OSGKeyboard · Shared
//
// Deterministic merge rules for iCloud KVS sync. Pure logic no
// NSUbiquitousKeyValueStore dependency so unit tests stay hermetic.
import Foundation
extension PersonalDictionary {
/// Merges two dictionary snapshots for cross-device sync.
///
/// Rules:
/// - Same `id`: keep the entry with the newer `updatedAt`.
/// - Same canonical term (case-insensitive) but different `id`: union
/// aliases, take max `usageCount`, keep the newer entry's fields.
public static func merge(local: PersonalDictionary, remote: PersonalDictionary) -> PersonalDictionary {
var mergedByID: [UUID: Entry] = [:]
var canonicalOwner: [String: UUID] = [:]
func insertOrMerge(_ candidate: Entry) {
let key = candidate.term.lowercased()
if let existingID = canonicalOwner[key], var existing = mergedByID[existingID] {
if candidate.id == existingID {
mergedByID[existingID] = resolveEntryConflict(existing: existing, incoming: candidate)
return
}
existing = mergeSameTerm(existing: existing, incoming: candidate)
mergedByID[existingID] = existing
return
}
if let existing = mergedByID[candidate.id] {
mergedByID[candidate.id] = resolveEntryConflict(existing: existing, incoming: candidate)
canonicalOwner[key] = candidate.id
return
}
mergedByID[candidate.id] = candidate
canonicalOwner[key] = candidate.id
}
for entry in local.entries { insertOrMerge(entry) }
for entry in remote.entries { insertOrMerge(entry) }
let mergedEntries = mergedByID.values.sorted {
if $0.updatedAt != $1.updatedAt {
return $0.updatedAt > $1.updatedAt
}
return $0.term.localizedCaseInsensitiveCompare($1.term) == .orderedAscending
}
let lastSyncedAt = [local.lastSyncedAt, remote.lastSyncedAt]
.compactMap { $0 }
.max()
return PersonalDictionary(
entries: mergedEntries,
version: max(local.version, remote.version) + 1,
lastSyncedAt: lastSyncedAt
)
}
private static func resolveEntryConflict(existing: Entry, incoming: Entry) -> Entry {
incoming.updatedAt >= existing.updatedAt ? incoming : existing
}
private static func mergeSameTerm(existing: Entry, incoming: Entry) -> Entry {
let winner = incoming.updatedAt >= existing.updatedAt ? incoming : existing
let loser = winner.id == existing.id ? incoming : existing
var merged = winner
merged.aliases = unionAliases(
existing: winner.aliases,
incoming: loser.aliases,
excludingTerm: winner.term
)
merged.usageCount = max(winner.usageCount, loser.usageCount)
merged.updatedAt = max(winner.updatedAt, loser.updatedAt)
return merged
}
private static func unionAliases(
existing: [String],
incoming: [String],
excludingTerm: String
) -> [String] {
let termLower = excludingTerm.lowercased()
return Array(
Set((existing + incoming).filter { $0.lowercased() != termLower })
).sorted { $0.localizedCaseInsensitiveCompare($1) == .orderedAscending }
}
}
@@ -21,10 +21,33 @@ import Foundation
public struct PersonalDictionary: Codable, Sendable, Equatable {
public var entries: [Entry]
public var version: Int
/// When this dictionary blob was last successfully pushed to iCloud KVS.
public var lastSyncedAt: Date?
public init(entries: [Entry] = [], version: Int = 1) {
public init(entries: [Entry] = [], version: Int = 1, lastSyncedAt: Date? = nil) {
self.entries = entries
self.version = version
self.lastSyncedAt = lastSyncedAt
}
private enum CodingKeys: String, CodingKey {
case entries
case version
case lastSyncedAt
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
entries = try container.decodeIfPresent([Entry].self, forKey: .entries) ?? []
version = try container.decodeIfPresent(Int.self, forKey: .version) ?? 1
lastSyncedAt = try container.decodeIfPresent(Date.self, forKey: .lastSyncedAt)
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(entries, forKey: .entries)
try container.encode(version, forKey: .version)
try container.encodeIfPresent(lastSyncedAt, forKey: .lastSyncedAt)
}
public struct Entry: Codable, Sendable, Equatable, Identifiable {
@@ -34,6 +57,8 @@ public struct PersonalDictionary: Codable, Sendable, Equatable {
public var category: Category
public var source: Source
public var createdAt: Date
/// Last mutation time used for iCloud merge conflict resolution.
public var updatedAt: Date
public var usageCount: Int
public init(
@@ -43,6 +68,7 @@ public struct PersonalDictionary: Codable, Sendable, Equatable {
category: Category,
source: Source,
createdAt: Date = Date(),
updatedAt: Date? = nil,
usageCount: Int = 0
) {
self.id = id
@@ -51,9 +77,45 @@ public struct PersonalDictionary: Codable, Sendable, Equatable {
self.category = category
self.source = source
self.createdAt = createdAt
self.updatedAt = updatedAt ?? createdAt
self.usageCount = usageCount
}
private enum CodingKeys: String, CodingKey {
case id
case term
case aliases
case category
case source
case createdAt
case updatedAt
case usageCount
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
id = try container.decode(UUID.self, forKey: .id)
term = try container.decode(String.self, forKey: .term)
aliases = try container.decodeIfPresent([String].self, forKey: .aliases) ?? []
category = try container.decode(Category.self, forKey: .category)
source = try container.decode(Source.self, forKey: .source)
createdAt = try container.decodeIfPresent(Date.self, forKey: .createdAt) ?? Date()
updatedAt = try container.decodeIfPresent(Date.self, forKey: .updatedAt) ?? createdAt
usageCount = try container.decodeIfPresent(Int.self, forKey: .usageCount) ?? 0
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(id, forKey: .id)
try container.encode(term, forKey: .term)
try container.encode(aliases, forKey: .aliases)
try container.encode(category, forKey: .category)
try container.encode(source, forKey: .source)
try container.encode(createdAt, forKey: .createdAt)
try container.encode(updatedAt, forKey: .updatedAt)
try container.encode(usageCount, forKey: .usageCount)
}
public enum Category: String, Codable, Sendable, CaseIterable {
/// Person / place / brand / organization.
case properNoun
@@ -140,6 +202,7 @@ extension PersonalDictionary {
category: .productName,
source: .manual,
createdAt: Date(timeIntervalSince1970: 0),
updatedAt: Date(timeIntervalSince1970: 0),
usageCount: 0
),
]
@@ -182,6 +245,7 @@ extension PersonalDictionary {
if termChanged || regenerateAliases {
entry.aliases = []
}
entry.updatedAt = Date()
entries[idx] = entry
return entry
}
@@ -193,15 +257,19 @@ extension PersonalDictionary {
entry.term = trimmed
entry.category = category
entry.source = .manual
entry.updatedAt = Date()
entries[idx] = entry
return entry
}
let now = Date()
let entry = Entry(
term: trimmed,
aliases: [],
category: category,
source: .manual
source: .manual,
createdAt: now,
updatedAt: now
)
entries.append(entry)
return entry
@@ -216,6 +284,7 @@ extension PersonalDictionary {
entries[idx].aliases = Array(
Set(cleaned.filter { $0.lowercased() != termLower })
).sorted { $0.localizedCaseInsensitiveCompare($1) == .orderedAscending }
entries[idx].updatedAt = Date()
}
/// Renders the entire dictionary as a prompt fragment. Entries
@@ -64,7 +64,7 @@ public final class ProviderConfig: ObservableObject, @unchecked Sendable {
}
}
/// "local" on-device ASR + built-in DeepSeek polish.
/// "cloud" on-device ASR + user's cloud LLM polish.
/// "cloud" provider cloud ASR (with personal dictionary) + user's cloud LLM polish.
@Published public var engineMode: String {
didSet {
guard !isApplyingConfiguration, engineMode != configuration.engineMode else { return }
@@ -1,13 +1,13 @@
{
"bin_bytes" : 174285,
"bin_bytes" : 198494,
"bin_file" : "OSGKeyboardCLM.bin",
"export_seconds" : 0.029847979545593262,
"generated_at" : "2026-07-05T11:34:09Z",
"export_seconds" : 0.035165071487426758,
"generated_at" : "2026-07-06T13:16:27Z",
"identifier" : "com.osgkeyboard.custom-lm.v1",
"locale" : "zh_CN",
"phrase_count" : 11550,
"phrase_count" : 13329,
"sources" : {
"ai_tech_seed" : 1259,
"ai_tech_seed" : 3040,
"computer_terms" : 10300
},
"version" : "1.0.0"
+7 -3
View File
@@ -121,9 +121,13 @@ public enum ASREvent: Sendable, Equatable {
// MARK: - Factory
public enum ASRServiceFactory {
/// Returns the on-device `SpeechAnalyzer` + `DictationTranscriber` backend.
public static func make() -> ASRService {
SpeechAnalyzerASR()
/// Returns on-device SpeechAnalyzer for `local`, or the user's cloud
/// ASR provider when `engineMode == "cloud"`.
public static func make(store: AppGroupStore = AppGroupStore()) -> ASRService {
if store.engineMode == "cloud" {
return CloudASRService(store: store)
}
return SpeechAnalyzerASR()
}
}
@@ -155,6 +155,16 @@ public struct AppGroupStore: @unchecked Sendable {
public func setPersonalDictionary(_ dictionary: PersonalDictionary) {
mutateConfiguration { $0.personalDictionary = dictionary }
AppGroupConfigDarwin.postConfigChanged()
}
public var personalDictionaryICloudSyncEnabled: Bool {
get { configuration.personalDictionaryICloudSyncEnabled }
set { setPersonalDictionaryICloudSyncEnabled(newValue) }
}
public func setPersonalDictionaryICloudSyncEnabled(_ enabled: Bool) {
mutateConfiguration { $0.personalDictionaryICloudSyncEnabled = enabled }
}
// MARK: - Client
@@ -0,0 +1,169 @@
// AlibabaVocabularySync.swift
// OSGKeyboard · Shared
//
// Syncs PersonalDictionary DashScope custom vocabulary (Fun-ASR Flash).
import Foundation
public enum AlibabaVocabularySync {
public enum Keys {
public static let vocabularyId = "config.alibabaASRVocabularyId"
public static let fingerprint = "config.alibabaASRVocabularyFingerprint"
}
private static let vocabularyPrefix = "osgkb"
/// Returns a ready `vocabulary_id`, creating or updating the remote list as needed.
public static func ensureVocabularyID(
dictionary: PersonalDictionary,
apiKey: String,
targetModel: String = CloudASRModelCatalog.alibabaVocabularyTargetModel,
defaults: UserDefaults,
session: URLSession = .shared
) async throws -> String? {
let entries = dictionary.alibabaHotwordEntries()
guard !entries.isEmpty else {
clearCache(defaults: defaults)
return nil
}
let fingerprint = dictionary.vocabularySyncFingerprint()
if let cachedID = defaults.string(forKey: Keys.vocabularyId),
defaults.string(forKey: Keys.fingerprint) == fingerprint,
!cachedID.isEmpty {
return cachedID
}
let url = try customizationURL()
if let existingID = defaults.string(forKey: Keys.vocabularyId), !existingID.isEmpty {
try await updateVocabulary(
id: existingID,
entries: entries,
apiKey: apiKey,
url: url,
session: session
)
cache(id: existingID, fingerprint: fingerprint, defaults: defaults)
return existingID
}
let createdID = try await createVocabulary(
entries: entries,
targetModel: targetModel,
apiKey: apiKey,
url: url,
session: session
)
cache(id: createdID, fingerprint: fingerprint, defaults: defaults)
return createdID
}
public static func clearCache(defaults: UserDefaults) {
defaults.removeObject(forKey: Keys.vocabularyId)
defaults.removeObject(forKey: Keys.fingerprint)
}
private static func cache(id: String, fingerprint: String, defaults: UserDefaults) {
defaults.set(id, forKey: Keys.vocabularyId)
defaults.set(fingerprint, forKey: Keys.fingerprint)
}
private static func customizationURL() throws -> URL {
let raw = CloudASRModelCatalog.alibabaAPIBase + CloudASRModelCatalog.alibabaCustomizationPath
guard let url = URL(string: raw) else { throw CloudASRError.invalidURL }
return url
}
private static func createVocabulary(
entries: [AlibabaHotwordEntry],
targetModel: String,
apiKey: String,
url: URL,
session: URLSession
) async throws -> String {
let vocabulary = entries.map { entry -> [String: Any] in
var item: [String: Any] = ["text": entry.text, "weight": entry.weight]
if let lang = entry.lang { item["lang"] = lang }
return item
}
let body: [String: Any] = [
"model": "speech-biasing",
"input": [
"action": "create_vocabulary",
"target_model": targetModel,
"prefix": vocabularyPrefix,
"vocabulary": vocabulary,
] as [String: Any],
]
let data = try await postJSON(body, to: url, apiKey: apiKey, session: session)
guard let id = parseVocabularyID(from: data) else {
throw CloudASRError.decoding("missing vocabulary_id")
}
return id
}
private static func updateVocabulary(
id: String,
entries: [AlibabaHotwordEntry],
apiKey: String,
url: URL,
session: URLSession
) async throws {
let vocabulary = entries.map { entry -> [String: Any] in
var item: [String: Any] = ["text": entry.text, "weight": entry.weight]
if let lang = entry.lang { item["lang"] = lang }
return item
}
let body: [String: Any] = [
"model": "speech-biasing",
"input": [
"action": "update_vocabulary",
"vocabulary_id": id,
"vocabulary": vocabulary,
] as [String: Any],
]
_ = try await postJSON(body, to: url, apiKey: apiKey, session: session)
}
private static func postJSON(
_ body: [String: Any],
to url: URL,
apiKey: String,
session: URLSession
) async throws -> [String: Any] {
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
request.httpBody = try JSONSerialization.data(withJSONObject: body)
let (data, response) = try await session.data(for: request)
guard let http = response as? HTTPURLResponse else {
throw CloudASRError.transport("non-HTTP response")
}
let json = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any]
guard (200..<300).contains(http.statusCode) else {
let message = parseAPIErrorMessage(from: json)
throw CloudASRError.http(status: http.statusCode, message: message)
}
return json ?? [:]
}
private static func parseVocabularyID(from json: [String: Any]) -> String? {
if let output = json["output"] as? [String: Any],
let id = output["vocabulary_id"] as? String {
return id
}
return nil
}
private static func parseAPIErrorMessage(from json: [String: Any]?) -> String? {
guard let json else { return nil }
if let message = json["message"] as? String { return message }
if let error = json["error"] as? [String: Any],
let message = error["message"] as? String {
return message
}
return nil
}
}
@@ -0,0 +1,411 @@
// CloudASRClients.swift
// OSGKeyboard · Shared
//
// Provider-specific cloud ASR backends with personal-dictionary bias.
import Foundation
public protocol CloudASRTranscribing: Sendable {
func prepare(dictionary: PersonalDictionary) async throws
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String
}
public enum CloudASRClientFactory {
public static func make(store: AppGroupStore, session: URLSession = .shared) -> CloudASRTranscribing {
let strategy = CloudASRModelCatalog.strategy(for: store.providerId)
switch strategy {
case .zhipuHotwords:
return ZhipuCloudASRClient(
apiKey: store.apiKey,
model: CloudASRModelCatalog.defaultModel(for: store.providerId),
session: session
)
case .alibabaVocabulary:
return AlibabaFunASRClient(
apiKey: store.apiKey,
model: CloudASRModelCatalog.defaultModel(for: store.providerId),
store: store,
session: session
)
case .prompt:
return PromptCloudASRClient(
providerId: store.providerId,
baseURL: store.baseURL,
apiKey: store.apiKey,
model: CloudASRModelCatalog.defaultModel(for: store.providerId),
session: session
)
case .localFallback:
return UnsupportedCloudASRClient(providerId: store.providerId)
}
}
}
// MARK: - Zhipu (hotwords + prompt)
struct ZhipuCloudASRClient: CloudASRTranscribing {
let apiKey: String
let model: String
let session: URLSession
private static let maxDurationSeconds: TimeInterval = 30
func prepare(dictionary: PersonalDictionary) async throws {}
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String {
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
let duration = Double(samples.count) / Double(sampleRate)
guard duration <= Self.maxDurationSeconds else { throw CloudASRError.audioTooLong }
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
let urlString = "https://open.bigmodel.cn/api/paas/v4\(CloudASRModelCatalog.zhipuTranscriptionPath)"
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
let boundary = "Boundary-\(UUID().uuidString)"
var body = Data()
func appendField(_ name: String, _ value: String) {
body.append("--\(boundary)\r\n".data(using: .utf8)!)
body.append("Content-Disposition: form-data; name=\"\(name)\"\r\n\r\n".data(using: .utf8)!)
body.append("\(value)\r\n".data(using: .utf8)!)
}
appendField("model", model)
appendField("stream", "false")
let hotwords = dictionary.asrHotwords()
if !hotwords.isEmpty,
let hotwordsJSON = try? JSONSerialization.data(withJSONObject: hotwords),
let hotwordsString = String(data: hotwordsJSON, encoding: .utf8) {
appendField("hotwords", hotwordsString)
}
let prompt = dictionary.asrPromptBias()
if !prompt.isEmpty {
appendField("prompt", prompt)
}
body.append("--\(boundary)\r\n".data(using: .utf8)!)
body.append("Content-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\n".data(using: .utf8)!)
body.append("Content-Type: audio/wav\r\n\r\n".data(using: .utf8)!)
body.append(wav)
body.append("\r\n".data(using: .utf8)!)
body.append("--\(boundary)--\r\n".data(using: .utf8)!)
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")
request.httpBody = body
request.timeoutInterval = 60
let (data, response) = try await session.data(for: request)
try Self.validateHTTP(response: response, data: data)
guard let text = Self.parseZhipuText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
!text.isEmpty else {
throw CloudASRError.emptyTranscript
}
return text
}
private static func parseZhipuText(from data: Data) -> String? {
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
return nil
}
return json["text"] as? String
}
fileprivate static func validateHTTP(response: URLResponse, data: Data) throws {
guard let http = response as? HTTPURLResponse else {
throw CloudASRError.transport("non-HTTP response")
}
guard (200..<300).contains(http.statusCode) else {
let message = parseErrorMessage(from: data)
throw CloudASRError.http(status: http.statusCode, message: message)
}
}
fileprivate static func parseErrorMessage(from data: Data) -> String? {
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
return nil
}
if let message = json["message"] as? String { return message }
if let error = json["error"] as? [String: Any],
let message = error["message"] as? String {
return message
}
return nil
}
}
// MARK: - Alibaba Fun-ASR Flash (vocabulary_id + context)
struct AlibabaFunASRClient: CloudASRTranscribing {
let apiKey: String
let model: String
// Hold the (@unchecked Sendable) AppGroupStore rather than a raw
// UserDefaults so this struct stays Sendable under strict concurrency.
let store: AppGroupStore
let session: URLSession
func prepare(dictionary: PersonalDictionary) async throws {
_ = try await AlibabaVocabularySync.ensureVocabularyID(
dictionary: dictionary,
apiKey: apiKey,
targetModel: CloudASRModelCatalog.alibabaVocabularyTargetModel,
defaults: store.defaults,
session: session
)
}
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String {
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
let vocabularyID = try await AlibabaVocabularySync.ensureVocabularyID(
dictionary: dictionary,
apiKey: apiKey,
targetModel: CloudASRModelCatalog.alibabaVocabularyTargetModel,
defaults: store.defaults,
session: session
)
let dataURI = PCMSampleWavEncoder.dataURI(samples: samples, sampleRate: sampleRate)
let urlString = CloudASRModelCatalog.alibabaAPIBase + CloudASRModelCatalog.alibabaMultimodalPath
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
var messages: [[String: Any]] = []
let context = dictionary.alibabaContextText()
if !context.isEmpty {
messages.append([
"role": "user",
"content": [
["type": "input_text", "text": context],
],
])
}
messages.append([
"role": "user",
"content": [
[
"type": "input_audio",
"input_audio": ["data": dataURI],
],
],
])
var parameters: [String: Any] = [
"format": "wav",
"sample_rate": "\(sampleRate)",
]
if let vocabularyID, !vocabularyID.isEmpty {
parameters["vocabulary_id"] = vocabularyID
}
let body: [String: Any] = [
"model": model,
"input": ["messages": messages],
"parameters": parameters,
]
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
request.setValue("disable", forHTTPHeaderField: "X-DashScope-SSE")
request.httpBody = try JSONSerialization.data(withJSONObject: body)
request.timeoutInterval = 90
let (data, response) = try await session.data(for: request)
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
guard let text = Self.parseText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
!text.isEmpty else {
throw CloudASRError.emptyTranscript
}
return text
}
private static func parseText(from data: Data) -> String? {
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
let output = json["output"] as? [String: Any] else {
return nil
}
if let text = output["text"] as? String { return text }
if let sentence = output["sentence"] as? [String: Any],
let text = sentence["text"] as? String {
return text
}
return nil
}
}
// MARK: - Prompt-biased transcription (OpenAI / MiMo / custom)
struct PromptCloudASRClient: CloudASRTranscribing {
let providerId: String
let baseURL: String
let apiKey: String
let model: String
let session: URLSession
func prepare(dictionary: PersonalDictionary) async throws {}
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String {
guard !apiKey.isEmpty else { throw CloudASRError.noAPIKey }
if providerId == "mimo" {
return try await transcribeMiMo(
samples: samples,
sampleRate: sampleRate,
dictionary: dictionary
)
}
return try await transcribeOpenAIStyle(
samples: samples,
sampleRate: sampleRate,
dictionary: dictionary
)
}
private func transcribeOpenAIStyle(
samples: [Float],
sampleRate: Int,
dictionary: PersonalDictionary
) async throws -> String {
let wav = PCMSampleWavEncoder.encode(samples: samples, sampleRate: sampleRate)
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
let urlString = "\(trimmedBase)/audio/transcriptions"
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
let boundary = "Boundary-\(UUID().uuidString)"
var body = Data()
func appendField(_ name: String, _ value: String) {
body.append("--\(boundary)\r\n".data(using: .utf8)!)
body.append("Content-Disposition: form-data; name=\"\(name)\"\r\n\r\n".data(using: .utf8)!)
body.append("\(value)\r\n".data(using: .utf8)!)
}
appendField("model", model)
let prompt = dictionary.asrPromptBias(maxCharacters: 600)
if !prompt.isEmpty {
appendField("prompt", prompt)
}
body.append("--\(boundary)\r\n".data(using: .utf8)!)
body.append("Content-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\n".data(using: .utf8)!)
body.append("Content-Type: audio/wav\r\n\r\n".data(using: .utf8)!)
body.append(wav)
body.append("\r\n".data(using: .utf8)!)
body.append("--\(boundary)--\r\n".data(using: .utf8)!)
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")
request.httpBody = body
request.timeoutInterval = 90
let (data, response) = try await session.data(for: request)
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
guard let text = Self.parseOpenAIText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
!text.isEmpty else {
throw CloudASRError.emptyTranscript
}
return text
}
private func transcribeMiMo(
samples: [Float],
sampleRate: Int,
dictionary: PersonalDictionary
) async throws -> String {
let dataURI = PCMSampleWavEncoder.dataURI(samples: samples, sampleRate: sampleRate)
let trimmedBase = baseURL.hasSuffix("/") ? String(baseURL.dropLast()) : baseURL
let urlString = "\(trimmedBase)/chat/completions"
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
var userContent: [[String: Any]] = []
let prompt = dictionary.asrPromptBias()
if !prompt.isEmpty {
userContent.append(["type": "text", "text": prompt])
}
userContent.append([
"type": "input_audio",
"input_audio": ["data": dataURI],
])
let body: [String: Any] = [
"model": model,
"messages": [
["role": "user", "content": userContent],
],
"asr_options": ["language": "auto"],
]
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
request.setValue(apiKey, forHTTPHeaderField: "api-key")
request.httpBody = try JSONSerialization.data(withJSONObject: body)
request.timeoutInterval = 90
let (data, response) = try await session.data(for: request)
try ZhipuCloudASRClient.validateHTTP(response: response, data: data)
guard let text = Self.parseChatCompletionText(from: data)?.trimmingCharacters(in: .whitespacesAndNewlines),
!text.isEmpty else {
throw CloudASRError.emptyTranscript
}
return text
}
private static func parseOpenAIText(from data: Data) -> String? {
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
return nil
}
return json["text"] as? String
}
private static func parseChatCompletionText(from data: Data) -> String? {
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
let choices = json["choices"] as? [[String: Any]],
let first = choices.first,
let message = first["message"] as? [String: Any] else {
return nil
}
return message["content"] as? String
}
}
// MARK: - Unsupported hosted ASR (Moonshot)
struct UnsupportedCloudASRClient: CloudASRTranscribing {
let providerId: String
func prepare(dictionary: PersonalDictionary) async throws {}
func transcribe(
samples: [Float],
sampleRate: Int,
locale: Locale,
dictionary: PersonalDictionary
) async throws -> String {
throw CloudASRError.providerUnsupported
}
}
@@ -0,0 +1,147 @@
// CloudASRService.swift
// OSGKeyboard · Shared
//
// Cloud-engine ASR: uploads PCM chunks to the user's configured provider
// with personal-dictionary bias. Moonshot falls back to on-device ASR.
import Foundation
import os
public final class CloudASRService: ASRService, @unchecked Sendable {
private let store: AppGroupStore
private let session: URLSession
private let localFallback: ASRService
private let lock = OSAllocatedUnfairLock()
private var client: CloudASRTranscribing?
private var usesLocalFallback = false
private var boundProviderId: String?
private var cancelled = false
public init(
store: AppGroupStore = AppGroupStore(),
session: URLSession = .shared,
localFallback: ASRService? = nil
) {
self.store = store
self.session = session
// `SpeechAnalyzerASR` is internal, so it can't appear in a public
// default argument value resolve the fallback in the body instead.
self.localFallback = localFallback ?? SpeechAnalyzerASR()
}
public func resetForNewUtterance() {
lock.withLock { cancelled = false }
if usesLocalFallback {
localFallback.resetForNewUtterance()
}
}
public func warmup(locale: Locale) async {
bindClientIfNeeded()
if usesLocalFallback {
await localFallback.warmup(locale: locale)
return
}
guard let client = lock.withLock({ client }) else { return }
do {
try await client.prepare(dictionary: store.personalDictionary)
} catch {
OSGLog.asr.warning("cloud ASR vocabulary prepare failed: \(error.localizedDescription, privacy: .public)")
}
}
public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
guard !samples.isEmpty else { return .success("") }
if Task.isCancelled || lock.withLock({ cancelled }) { return .cancelled }
bindClientIfNeeded()
if usesLocalFallback {
return await localFallback.transcribeChunk(samples: samples, locale: locale)
}
guard let client = lock.withLock({ client }) else {
return .failure(CloudASRError.providerUnsupported.localizedDescription)
}
do {
let text = try await client.transcribe(
samples: samples,
sampleRate: 16_000,
locale: locale,
dictionary: store.personalDictionary
)
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? .success("") : .success(trimmed)
} catch is CancellationError {
return .cancelled
} catch {
return .failure(error.localizedDescription)
}
}
public func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent> {
bindClientIfNeeded()
if usesLocalFallback {
return localFallback.transcribe(stream: stream, locale: locale)
}
return AsyncStream { continuation in
continuation.yield(.capability(onDeviceSupported: false))
let task = Task {
var samples: [Float] = []
for await snap in stream {
if Task.isCancelled { break }
samples.append(contentsOf: snap.samples)
}
guard !Task.isCancelled, !self.lock.withLock({ self.cancelled }) else {
continuation.finish()
return
}
guard !samples.isEmpty else {
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
continuation.finish()
return
}
switch await self.transcribeChunk(samples: samples, locale: locale) {
case .success(let text):
if text.isEmpty {
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
} else {
continuation.yield(.final(text))
}
case .failure(let message):
continuation.yield(.error(message))
case .cancelled:
break
}
continuation.finish()
}
continuation.onTermination = { @Sendable _ in
task.cancel()
self.cancel()
}
}
}
public func cancel() {
lock.withLock { cancelled = true }
localFallback.cancel()
}
private func bindClientIfNeeded() {
let providerId = store.providerId
let strategy = CloudASRModelCatalog.strategy(for: providerId)
lock.withLock {
guard boundProviderId != providerId else { return }
boundProviderId = providerId
usesLocalFallback = strategy == .localFallback
client = usesLocalFallback
? nil
: CloudASRClientFactory.make(store: store, session: session)
}
}
}
@@ -115,7 +115,7 @@ public final class LiveDictationController: ObservableObject {
private var didInstallTap = false
public init(asr: ASRService? = nil) {
self.asr = asr ?? ASRServiceFactory.make()
self.asr = asr ?? ASRServiceFactory.make(store: AppGroupStore())
}
/// Start dictation using a persisted settings locale id (`auto`, `zh-Hans`, ).
@@ -0,0 +1,146 @@
// PersonalDictionaryCloudSync.swift
// OSGKeyboard · Shared
//
// Mirrors the personal dictionary through iCloud Key-Value Store while
// keeping App Group UserDefaults as the keyboard extension's runtime
// source of truth. Intended for main-app call sites only.
import Foundation
public extension Notification.Name {
/// Posted after a remote KVS pull updates the App Group dictionary.
static let personalDictionaryDidSyncFromCloud = Notification.Name(
"com.osgkeyboard.personalDictionary.didSyncFromCloud"
)
}
public enum PersonalDictionaryCloudSyncError: Error, Equatable, Sendable {
case payloadTooLarge(byteCount: Int)
case encodeFailed
case decodeFailed
}
@MainActor
public final class PersonalDictionaryCloudSync {
public static let shared = PersonalDictionaryCloudSync()
public static let kvsKey = "personalDictionary.v1"
/// Stay below the ~1 MB per-key KVS limit.
public static let maxPayloadBytes = 900_000
private let kvs: UbiquitousKeyValueStoreing
private let makeStore: () -> AppGroupStore
private var externalChangeObserver: NSObjectProtocol?
public init(
kvs: UbiquitousKeyValueStoreing = NSUbiquitousKeyValueStore.default,
makeStore: @escaping () -> AppGroupStore = { AppGroupStore() }
) {
self.kvs = kvs
self.makeStore = makeStore
}
// MARK: - Lifecycle
public func startObservingExternalChanges() {
guard externalChangeObserver == nil else { return }
externalChangeObserver = NotificationCenter.default.addObserver(
forName: NSUbiquitousKeyValueStore.didChangeExternallyNotification,
object: nil,
queue: .main
) { [weak self] _ in
guard let self else { return }
Task { @MainActor in
await self.pullAndMergeIfEnabled()
}
}
}
public func stopObservingExternalChanges() {
if let externalChangeObserver {
NotificationCenter.default.removeObserver(externalChangeObserver)
self.externalChangeObserver = nil
}
}
/// Pull remote changes on launch / foreground when sync is enabled.
public func pullAndMergeIfEnabled() async {
let store = makeStore()
guard store.personalDictionaryICloudSyncEnabled else { return }
await pullAndMerge(store: store)
}
/// Push the current local dictionary when sync is enabled.
public func pushLocalIfEnabled(_ dictionary: PersonalDictionary) async throws {
let store = makeStore()
guard store.personalDictionaryICloudSyncEnabled else { return }
try push(dictionary)
}
/// Enable sync: merge local + remote, persist locally, then upload.
public func enableSync() async throws {
let store = makeStore()
store.setPersonalDictionaryICloudSyncEnabled(true)
let local = store.personalDictionary
let remote = loadRemote() ?? .empty
let merged = PersonalDictionary.merge(local: local, remote: remote)
store.setPersonalDictionary(merged)
try push(merged)
}
public func disableSync() {
makeStore().setPersonalDictionaryICloudSyncEnabled(false)
}
// MARK: - Core operations
public func pullAndMerge(store: AppGroupStore) async {
guard store.personalDictionaryICloudSyncEnabled else { return }
let local = store.personalDictionary
guard let remote = loadRemote() else { return }
let merged = PersonalDictionary.merge(local: local, remote: remote)
guard merged != local else { return }
store.setPersonalDictionary(merged)
NotificationCenter.default.post(name: .personalDictionaryDidSyncFromCloud, object: nil)
}
public func push(_ dictionary: PersonalDictionary) throws {
var payload = dictionary
payload.lastSyncedAt = Date()
let data = try encode(payload)
kvs.set(data, forKey: Self.kvsKey)
_ = kvs.synchronize()
}
public func loadRemote() -> PersonalDictionary? {
guard let data = kvs.data(forKey: Self.kvsKey) else { return nil }
return try? decode(data)
}
// MARK: - Encoding
public func encode(_ dictionary: PersonalDictionary) throws -> Data {
let encoder = JSONEncoder()
encoder.dateEncodingStrategy = .iso8601
guard let data = try? encoder.encode(dictionary) else {
throw PersonalDictionaryCloudSyncError.encodeFailed
}
guard data.count <= Self.maxPayloadBytes else {
throw PersonalDictionaryCloudSyncError.payloadTooLarge(byteCount: data.count)
}
return data
}
public func decode(_ data: Data) throws -> PersonalDictionary {
let decoder = JSONDecoder()
decoder.dateDecodingStrategy = .iso8601
guard let dictionary = try? decoder.decode(PersonalDictionary.self, from: data) else {
throw PersonalDictionaryCloudSyncError.decodeFailed
}
return dictionary
}
}
@@ -0,0 +1,15 @@
// UbiquitousKeyValueStoreing.swift
// OSGKeyboard · Shared
//
// Test seam around `NSUbiquitousKeyValueStore`.
import Foundation
public protocol UbiquitousKeyValueStoreing: AnyObject {
func data(forKey key: String) -> Data?
func set(_ value: Data?, forKey key: String)
@discardableResult
func synchronize() -> Bool
}
extension NSUbiquitousKeyValueStore: UbiquitousKeyValueStoreing {}
@@ -10,8 +10,8 @@
// English dictation while halving the network round-trip.
//
// Engine matrix:
// - `engineMode == "cloud"` on-device ASR, then user's cloud LLM
// - `engineMode == "local"` on-device ASR, then built-in DeepSeek
// - `engineMode == "cloud"` provider cloud ASR + user's cloud LLM
// - `engineMode == "local"` on-device ASR + built-in DeepSeek
// - Ultra-short, structure-free utterances skip the LLM entirely
// - Cloud without API key raw + `.missingAPIKey` warning
// - Local without build key raw + `.missingAPIKey` warning
@@ -6,6 +6,7 @@
// or a safety timeout elapses (symmetric to pre-roll at utterance start).
import Foundation
import os
/// Tunable tail-drain policy shared by Flow capture and preview dictation.
public struct FlowCaptureTailDrainPolicy: Sendable, Equatable {
@@ -9,8 +9,7 @@ import os
public enum FlowPipelineDiagnostics {
public static func logDrain(_ report: FlowCaptureDrainReport) {
OSGLog.flow.info(
"tailDrain duration=\(report.drainDurationSeconds, format: .fixed(precision: 2))s " +
"silenceEnd=\(report.endedBySilence) tailSamples=\(report.tailSampleCount)"
"tailDrain duration=\(report.drainDurationSeconds, format: .fixed(precision: 2))s silenceEnd=\(report.endedBySilence) tailSamples=\(report.tailSampleCount)"
)
}
@@ -21,8 +20,7 @@ public enum FlowPipelineDiagnostics {
chunkWarnings: Int
) {
OSGLog.flow.info(
"chunkPipeline chunks=\(chunkCount) lastChunkSamples=\(lastChunkSamples) " +
"stitchedLen=\(stitchedLength) warnings=\(chunkWarnings)"
"chunkPipeline chunks=\(chunkCount) lastChunkSamples=\(lastChunkSamples) stitchedLen=\(stitchedLength) warnings=\(chunkWarnings)"
)
}
@@ -0,0 +1,67 @@
// PCMSampleWavEncoder.swift
// OSGKeyboard · Shared
//
// Encodes mono Float32 PCM (@ 16 kHz) into a minimal WAV byte stream for
// cloud ASR multipart / base64 uploads.
import Foundation
public enum PCMSampleWavEncoder {
public static func encode(samples: [Float], sampleRate: Int = 16_000) -> Data {
guard !samples.isEmpty else {
return encode(pcm16: [], sampleRate: sampleRate)
}
var pcm16 = [Int16]()
pcm16.reserveCapacity(samples.count)
for sample in samples {
let scaled = sample * 32_767.0
let clipped = Swift.max(-32_768.0, Swift.min(32_767.0, scaled))
pcm16.append(Int16(clipped.rounded()))
}
return encode(pcm16: pcm16, sampleRate: sampleRate)
}
public static func encode(pcm16: [Int16], sampleRate: Int) -> Data {
let byteRate = sampleRate * 2
let dataSize = pcm16.count * MemoryLayout<Int16>.size
var data = Data()
data.reserveCapacity(44 + dataSize)
func appendASCII(_ string: String) {
data.append(contentsOf: string.utf8)
}
func appendLE32(_ value: UInt32) {
var le = value.littleEndian
withUnsafeBytes(of: &le) { data.append(contentsOf: $0) }
}
func appendLE16(_ value: UInt16) {
var le = value.littleEndian
withUnsafeBytes(of: &le) { data.append(contentsOf: $0) }
}
appendASCII("RIFF")
appendLE32(UInt32(36 + dataSize))
appendASCII("WAVE")
appendASCII("fmt ")
appendLE32(16)
appendLE16(1) // PCM
appendLE16(1) // mono
appendLE32(UInt32(sampleRate))
appendLE32(UInt32(byteRate))
appendLE16(2) // block align
appendLE16(16) // bits per sample
appendASCII("data")
appendLE32(UInt32(dataSize))
pcm16.withUnsafeBufferPointer { buffer in
guard let base = buffer.baseAddress else { return }
data.append(UnsafeBufferPointer(start: base, count: buffer.count))
}
return data
}
public static func dataURI(samples: [Float], sampleRate: Int = 16_000) -> String {
let wav = encode(samples: samples, sampleRate: sampleRate)
return "data:audio/wav;base64,\(wav.base64EncodedString())"
}
}
+11
View File
@@ -33,6 +33,17 @@
"error.asr.noSpeech" = "No speech detected. Please try again.";
"error.asr.chunkFailed" = "Segment %lld failed: %@";
/* Cloud ASR errors */
"error.cloudASR.noAPIKey" = "API Key is missing.";
"error.cloudASR.invalidURL" = "Invalid cloud ASR endpoint.";
"error.cloudASR.http" = "Cloud ASR returned HTTP %lld.";
"error.cloudASR.httpWithMessage" = "Cloud ASR returned HTTP %lld: %@";
"error.cloudASR.decoding" = "Failed to parse cloud ASR response: %@";
"error.cloudASR.transport" = "Cloud ASR network error: %@";
"error.cloudASR.emptyTranscript" = "Cloud ASR returned an empty transcript.";
"error.cloudASR.audioTooLong" = "Audio segment is too long for this cloud ASR provider.";
"error.cloudASR.providerUnsupported" = "This provider does not support cloud speech recognition yet.";
/* Polish scenarios */
"polishScenario.daily_chat" = "Daily Chat";
"polishScenario.social_lifestyle" = "Social Network";
@@ -33,6 +33,17 @@
"error.asr.noSpeech" = "未识别到语音内容,请重试。";
"error.asr.chunkFailed" = "第 %lld 段识别失败:%@";
/* Cloud ASR errors */
"error.cloudASR.noAPIKey" = "未填写 API Key。";
"error.cloudASR.invalidURL" = "云端识别接口地址无效。";
"error.cloudASR.http" = "云端识别返回 HTTP %lld。";
"error.cloudASR.httpWithMessage" = "云端识别返回 HTTP %lld%@";
"error.cloudASR.decoding" = "解析云端识别响应失败:%@";
"error.cloudASR.transport" = "云端识别网络错误:%@";
"error.cloudASR.emptyTranscript" = "云端识别返回了空文本。";
"error.cloudASR.audioTooLong" = "音频片段超过该云端识别服务的时长限制。";
"error.cloudASR.providerUnsupported" = "该服务商暂不支持云端语音识别。";
/* 润色场景 */
"polishScenario.daily_chat" = "日常聊天";
"polishScenario.social_lifestyle" = "小红书";