Squashes the working tree that had accumulated on cloud-agent. Two strands of work, kept in one commit at the maintainer's request. Crash and stability (this session): - Subscribe to MetricKit so crash, hang, CPU, and disk-write diagnostics are recorded on device. Keyboard-extension memory kills are the most likely way a user loses the keyboard mid-sentence and iOS does not classify them as crashes, so they never reach App Store Connect. - Upload those reports to POST /v1/analytics/diagnostics from internal builds only (local Debug and TestFlight), reusing the analytics installation identifier and skipping entirely when analytics are off. App Store builds keep everything on device. - Shed the Chinese engine, English lexicon, and in-flight pipeline work at the 40 MiB and 48 MiB budget thresholds instead of waiting for a system memory warning that jetsam often preempts. - Install audio taps with the input node's live format through one shared adaptive resampler, so an audio-route change mid-recording no longer raises an uncatchable format-mismatch exception. Covers the dictation preview and the Mac recorder, which still used the pattern the Flow capture path had already moved away from. - Harden the librime bridge: never pass a nil string across a nonnull boundary when a user dictionary holds malformed UTF-8, null-check the engine API vector, and distrust a candidate count without its array. - Guard the expanded candidate grid against a list that shrinks while the collection view is still laying out against the previous count. - Resolve localized text without the App Group so the screen shown when that container is missing renders its message instead of crashing on launch, which is the one situation it exists to handle. - Gate CI on the App Group and Keychain wiring that the keyboard extension hard-depends on, with a self-test proving the check fails. Clipboard, reply, and semantics (pre-existing working tree): - Unify the reply center, refresh keyboard AI features, and add native event and reminder exporters. - Expand clipboard semantic routing with authored corpora and a product-blind holdout, plus the scripts that build and validate them. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
221 lines
8.5 KiB
Swift
221 lines
8.5 KiB
Swift
// CloudASRModels.swift
|
||
// OSGKeyboard · Shared
|
||
//
|
||
// Cloud-engine ASR routing: which provider uses official hotwords /
|
||
// vocabulary APIs vs. a transcription prompt bias.
|
||
|
||
import Foundation
|
||
|
||
/// How a cloud provider applies the user's personal dictionary during ASR.
|
||
public enum CloudASRStrategy: String, Sendable, Equatable {
|
||
/// 智谱 GLM-ASR — `hotwords` + optional `prompt`.
|
||
case zhipuHotwords
|
||
/// 百炼 Fun-ASR Realtime — DashScope 经典 inference WebSocket 流式。
|
||
case bailianStreaming
|
||
/// OpenAI / Groq / 硅基流动 / Whisper 等 — `prompt` on transcription APIs.
|
||
case prompt
|
||
/// OpenRouter `/audio/transcriptions` — JSON body + base64 WAV (not multipart).
|
||
case openRouterJson
|
||
/// 火山引擎 SAUC 大模型流式 ASR(WebSocket + binary frame)。
|
||
case volcengineStreaming
|
||
/// OpenAI Realtime transcription(WebSocket,真流式)。
|
||
case openaiRealtimeStreaming
|
||
/// Moonshot 托管 API 暂无音频转写;云端引擎回退端侧 ASR。
|
||
case localFallback
|
||
}
|
||
|
||
public enum CloudASRError: Error, LocalizedError, Sendable, Equatable {
|
||
case noAPIKey
|
||
case invalidURL
|
||
case http(status: Int, message: String?)
|
||
case decoding(String)
|
||
case transport(String)
|
||
case emptyTranscript
|
||
case audioTooLong
|
||
case providerUnsupported
|
||
case timedOut
|
||
|
||
public var errorDescription: String? {
|
||
switch self {
|
||
case .noAPIKey:
|
||
return SharedL10n.string("error.cloudASR.noAPIKey")
|
||
case .invalidURL:
|
||
return SharedL10n.string("error.cloudASR.invalidURL")
|
||
case .http(let status, let message):
|
||
// 402 / balance-related errors: SiliconFlow gates some ¥0-priced
|
||
// models behind a positive account balance. Surface an actionable
|
||
// hint instead of the raw provider string.
|
||
if Self.isInsufficientBalance(status: status, message: message) {
|
||
return SharedL10n.string("error.cloudASR.insufficientBalance")
|
||
}
|
||
if let message, !message.isEmpty {
|
||
return SharedL10n.format("error.cloudASR.httpWithMessage", status, message)
|
||
}
|
||
return SharedL10n.format("error.cloudASR.http", status)
|
||
case .decoding(let detail):
|
||
return SharedL10n.format("error.cloudASR.decoding", detail)
|
||
case .transport(let detail):
|
||
return SharedL10n.format("error.cloudASR.transport", detail)
|
||
case .emptyTranscript:
|
||
return SharedL10n.string("error.cloudASR.emptyTranscript")
|
||
case .audioTooLong:
|
||
return SharedL10n.string("error.cloudASR.audioTooLong")
|
||
case .providerUnsupported:
|
||
return SharedL10n.string("error.cloudASR.providerUnsupported")
|
||
case .timedOut:
|
||
return SharedL10n.string("error.cloudASR.timedOut")
|
||
}
|
||
}
|
||
|
||
/// Whether an HTTP failure is an account-balance / billing gate rather than
|
||
/// a bad key or missing model. Keyed off the 402 status or a message that
|
||
/// mentions balance/欠费, covering providers that reuse other 4xx codes.
|
||
static func isInsufficientBalance(status: Int, message: String?) -> Bool {
|
||
if status == 402 { return true }
|
||
guard let message = message?.lowercased() else { return false }
|
||
return message.contains("insufficient")
|
||
|| message.contains("balance")
|
||
|| message.contains("余额")
|
||
|| message.contains("欠费")
|
||
}
|
||
}
|
||
|
||
public enum CloudASRModelCatalog {
|
||
/// Provider ids shown in the cloud ASR picker (explicit allowlist).
|
||
public static let selectableProviderIds: Set<String> = [
|
||
"openai",
|
||
"whisper",
|
||
"bailian",
|
||
"zhipu",
|
||
"groq",
|
||
"siliconflow",
|
||
"openrouter",
|
||
"mimo",
|
||
"volcengine",
|
||
"custom"
|
||
]
|
||
|
||
/// Sync Fun-ASR Flash — base64 upload, ≤ 5 min, supports context + vocabulary.
|
||
public static let alibabaFunASRFlash = "fun-asr-flash-2026-06-15"
|
||
public static let alibabaFunASRRealtime = "fun-asr-realtime"
|
||
/// Must match the ASR model used at recognition time.
|
||
public static let alibabaVocabularyTargetModel = alibabaFunASRFlash
|
||
|
||
public static let zhipuGLMASR = "glm-asr-2512"
|
||
public static let openAITranscribe = "gpt-4o-mini-transcribe"
|
||
public static let openAIWhisper = "whisper-1"
|
||
/// OpenAI Realtime transcription model (utterance-level streaming).
|
||
public static let openAIRealtimeWhisper = "gpt-realtime-whisper"
|
||
public static let openAIRealtimeEndpoint = "wss://api.openai.com/v1/realtime?intent=transcription"
|
||
public static let mimoASR = "mimo-v2.5-asr"
|
||
public static let groqWhisper = "whisper-large-v3-turbo"
|
||
public static let siliconflowASR = "FunAudioLLM/SenseVoiceSmall"
|
||
public static let openrouterWhisper = "openai/whisper-large-v3-turbo"
|
||
public static let volcengineDefaultResourceID = "volc.seedasr.sauc.duration"
|
||
public static let volcengineEndpoint = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async"
|
||
public static let bailianDefaultEndpoint = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/"
|
||
|
||
public static let alibabaAPIBase = "https://dashscope.aliyuncs.com/api/v1"
|
||
public static let alibabaCustomizationPath = "/services/audio/asr/customization"
|
||
public static let alibabaMultimodalPath = "/services/aigc/multimodal-generation/generation"
|
||
public static let zhipuTranscriptionPath = "/audio/transcriptions"
|
||
|
||
public static func supportsCloudASRSelection(providerId: String) -> Bool {
|
||
selectableProviderIds.contains(providerId)
|
||
}
|
||
|
||
public static func strategy(for providerId: String) -> CloudASRStrategy {
|
||
switch providerId {
|
||
case "zhipu":
|
||
return .zhipuHotwords
|
||
case "bailian":
|
||
return .bailianStreaming
|
||
case "moonshot":
|
||
return .localFallback
|
||
case "volcengine":
|
||
return .volcengineStreaming
|
||
case "openai":
|
||
return .openaiRealtimeStreaming
|
||
case "openrouter":
|
||
return .openRouterJson
|
||
case "whisper", "mimo", "groq", "siliconflow", "custom":
|
||
return .prompt
|
||
default:
|
||
return .localFallback
|
||
}
|
||
}
|
||
|
||
/// Providers whose Flow path uses utterance-level true streaming ASR.
|
||
public static func supportsTrueStreamingASR(for providerId: String) -> Bool {
|
||
switch strategy(for: providerId) {
|
||
case .bailianStreaming, .volcengineStreaming, .openaiRealtimeStreaming:
|
||
return true
|
||
case .zhipuHotwords, .prompt, .openRouterJson, .localFallback:
|
||
return false
|
||
}
|
||
}
|
||
|
||
public static func defaultModel(for providerId: String) -> String {
|
||
switch providerId {
|
||
case "zhipu":
|
||
return zhipuGLMASR
|
||
case "bailian":
|
||
return alibabaFunASRRealtime
|
||
case "whisper":
|
||
return openAIWhisper
|
||
case "mimo":
|
||
return mimoASR
|
||
case "groq":
|
||
return groqWhisper
|
||
case "siliconflow":
|
||
return siliconflowASR
|
||
case "openrouter":
|
||
return openrouterWhisper
|
||
case "volcengine":
|
||
return volcengineDefaultResourceID
|
||
case "openai":
|
||
return openAIRealtimeWhisper
|
||
case "custom":
|
||
return openAITranscribe
|
||
default:
|
||
return openAITranscribe
|
||
}
|
||
}
|
||
|
||
/// Whether the ASR settings card should expose a custom endpoint field.
|
||
public static func showsASREndpointField(for providerId: String) -> Bool {
|
||
switch strategy(for: providerId) {
|
||
case .prompt, .openRouterJson, .bailianStreaming, .openaiRealtimeStreaming:
|
||
return true
|
||
case .zhipuHotwords, .volcengineStreaming, .localFallback:
|
||
return false
|
||
}
|
||
}
|
||
}
|
||
|
||
extension LLMProvider {
|
||
public var cloudASRStrategy: CloudASRStrategy {
|
||
CloudASRModelCatalog.strategy(for: id)
|
||
}
|
||
|
||
public var defaultCloudASRModel: String {
|
||
CloudASRModelCatalog.defaultModel(for: id)
|
||
}
|
||
|
||
/// Official hotwords / vocabulary APIs during cloud ASR (not prompt-only bias).
|
||
public var supportsPersonalDictionaryCloudASR: Bool {
|
||
switch cloudASRStrategy {
|
||
case .zhipuHotwords:
|
||
return true
|
||
case .bailianStreaming, .prompt, .openRouterJson, .volcengineStreaming,
|
||
.openaiRealtimeStreaming, .localFallback:
|
||
return false
|
||
}
|
||
}
|
||
|
||
/// Product badge: true streaming ASR path is wired for this provider.
|
||
public var supportsStreamingCloudASR: Bool {
|
||
CloudASRModelCatalog.supportsTrueStreamingASR(for: id)
|
||
}
|
||
}
|