Files
OSGKeyboard/OSGKeyboardShared/Models/CloudASRModels.swift
T
rockyandClaude Opus 5 0023741455 feat(app): add crash visibility, memory relief, and clipboard reply work
Squashes the working tree that had accumulated on cloud-agent. Two
strands of work, kept in one commit at the maintainer's request.

Crash and stability (this session):

- Subscribe to MetricKit so crash, hang, CPU, and disk-write diagnostics
  are recorded on device. Keyboard-extension memory kills are the most
  likely way a user loses the keyboard mid-sentence and iOS does not
  classify them as crashes, so they never reach App Store Connect.
- Upload those reports to POST /v1/analytics/diagnostics from internal
  builds only (local Debug and TestFlight), reusing the analytics
  installation identifier and skipping entirely when analytics are off.
  App Store builds keep everything on device.
- Shed the Chinese engine, English lexicon, and in-flight pipeline work
  at the 40 MiB and 48 MiB budget thresholds instead of waiting for a
  system memory warning that jetsam often preempts.
- Install audio taps with the input node's live format through one
  shared adaptive resampler, so an audio-route change mid-recording no
  longer raises an uncatchable format-mismatch exception. Covers the
  dictation preview and the Mac recorder, which still used the pattern
  the Flow capture path had already moved away from.
- Harden the librime bridge: never pass a nil string across a nonnull
  boundary when a user dictionary holds malformed UTF-8, null-check the
  engine API vector, and distrust a candidate count without its array.
- Guard the expanded candidate grid against a list that shrinks while
  the collection view is still laying out against the previous count.
- Resolve localized text without the App Group so the screen shown when
  that container is missing renders its message instead of crashing on
  launch, which is the one situation it exists to handle.
- Gate CI on the App Group and Keychain wiring that the keyboard
  extension hard-depends on, with a self-test proving the check fails.

Clipboard, reply, and semantics (pre-existing working tree):

- Unify the reply center, refresh keyboard AI features, and add native
  event and reminder exporters.
- Expand clipboard semantic routing with authored corpora and a
  product-blind holdout, plus the scripts that build and validate them.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-18 17:31:29 -07:00

221 lines
8.5 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// CloudASRModels.swift
// OSGKeyboard · Shared
//
// Cloud-engine ASR routing: which provider uses official hotwords /
// vocabulary APIs vs. a transcription prompt bias.
import Foundation
/// How a cloud provider applies the user's personal dictionary during ASR.
public enum CloudASRStrategy: String, Sendable, Equatable {
/// 智谱 GLM-ASR — `hotwords` + optional `prompt`.
case zhipuHotwords
/// 百炼 Fun-ASR Realtime — DashScope 经典 inference WebSocket 流式。
case bailianStreaming
/// OpenAI / Groq / 硅基流动 / Whisper 等 — `prompt` on transcription APIs.
case prompt
/// OpenRouter `/audio/transcriptions` — JSON body + base64 WAV (not multipart).
case openRouterJson
/// 火山引擎 SAUC 大模型流式 ASR(WebSocket + binary frame)。
case volcengineStreaming
/// OpenAI Realtime transcription(WebSocket,真流式)。
case openaiRealtimeStreaming
/// Moonshot 托管 API 暂无音频转写;云端引擎回退端侧 ASR。
case localFallback
}
public enum CloudASRError: Error, LocalizedError, Sendable, Equatable {
case noAPIKey
case invalidURL
case http(status: Int, message: String?)
case decoding(String)
case transport(String)
case emptyTranscript
case audioTooLong
case providerUnsupported
case timedOut
public var errorDescription: String? {
switch self {
case .noAPIKey:
return SharedL10n.string("error.cloudASR.noAPIKey")
case .invalidURL:
return SharedL10n.string("error.cloudASR.invalidURL")
case .http(let status, let message):
// 402 / balance-related errors: SiliconFlow gates some ¥0-priced
// models behind a positive account balance. Surface an actionable
// hint instead of the raw provider string.
if Self.isInsufficientBalance(status: status, message: message) {
return SharedL10n.string("error.cloudASR.insufficientBalance")
}
if let message, !message.isEmpty {
return SharedL10n.format("error.cloudASR.httpWithMessage", status, message)
}
return SharedL10n.format("error.cloudASR.http", status)
case .decoding(let detail):
return SharedL10n.format("error.cloudASR.decoding", detail)
case .transport(let detail):
return SharedL10n.format("error.cloudASR.transport", detail)
case .emptyTranscript:
return SharedL10n.string("error.cloudASR.emptyTranscript")
case .audioTooLong:
return SharedL10n.string("error.cloudASR.audioTooLong")
case .providerUnsupported:
return SharedL10n.string("error.cloudASR.providerUnsupported")
case .timedOut:
return SharedL10n.string("error.cloudASR.timedOut")
}
}
/// Whether an HTTP failure is an account-balance / billing gate rather than
/// a bad key or missing model. Keyed off the 402 status or a message that
/// mentions balance/欠费, covering providers that reuse other 4xx codes.
static func isInsufficientBalance(status: Int, message: String?) -> Bool {
if status == 402 { return true }
guard let message = message?.lowercased() else { return false }
return message.contains("insufficient")
|| message.contains("balance")
|| message.contains("余额")
|| message.contains("欠费")
}
}
public enum CloudASRModelCatalog {
/// Provider ids shown in the cloud ASR picker (explicit allowlist).
public static let selectableProviderIds: Set<String> = [
"openai",
"whisper",
"bailian",
"zhipu",
"groq",
"siliconflow",
"openrouter",
"mimo",
"volcengine",
"custom"
]
/// Sync Fun-ASR Flash — base64 upload, ≤ 5 min, supports context + vocabulary.
public static let alibabaFunASRFlash = "fun-asr-flash-2026-06-15"
public static let alibabaFunASRRealtime = "fun-asr-realtime"
/// Must match the ASR model used at recognition time.
public static let alibabaVocabularyTargetModel = alibabaFunASRFlash
public static let zhipuGLMASR = "glm-asr-2512"
public static let openAITranscribe = "gpt-4o-mini-transcribe"
public static let openAIWhisper = "whisper-1"
/// OpenAI Realtime transcription model (utterance-level streaming).
public static let openAIRealtimeWhisper = "gpt-realtime-whisper"
public static let openAIRealtimeEndpoint = "wss://api.openai.com/v1/realtime?intent=transcription"
public static let mimoASR = "mimo-v2.5-asr"
public static let groqWhisper = "whisper-large-v3-turbo"
public static let siliconflowASR = "FunAudioLLM/SenseVoiceSmall"
public static let openrouterWhisper = "openai/whisper-large-v3-turbo"
public static let volcengineDefaultResourceID = "volc.seedasr.sauc.duration"
public static let volcengineEndpoint = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async"
public static let bailianDefaultEndpoint = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/"
public static let alibabaAPIBase = "https://dashscope.aliyuncs.com/api/v1"
public static let alibabaCustomizationPath = "/services/audio/asr/customization"
public static let alibabaMultimodalPath = "/services/aigc/multimodal-generation/generation"
public static let zhipuTranscriptionPath = "/audio/transcriptions"
public static func supportsCloudASRSelection(providerId: String) -> Bool {
selectableProviderIds.contains(providerId)
}
public static func strategy(for providerId: String) -> CloudASRStrategy {
switch providerId {
case "zhipu":
return .zhipuHotwords
case "bailian":
return .bailianStreaming
case "moonshot":
return .localFallback
case "volcengine":
return .volcengineStreaming
case "openai":
return .openaiRealtimeStreaming
case "openrouter":
return .openRouterJson
case "whisper", "mimo", "groq", "siliconflow", "custom":
return .prompt
default:
return .localFallback
}
}
/// Providers whose Flow path uses utterance-level true streaming ASR.
public static func supportsTrueStreamingASR(for providerId: String) -> Bool {
switch strategy(for: providerId) {
case .bailianStreaming, .volcengineStreaming, .openaiRealtimeStreaming:
return true
case .zhipuHotwords, .prompt, .openRouterJson, .localFallback:
return false
}
}
public static func defaultModel(for providerId: String) -> String {
switch providerId {
case "zhipu":
return zhipuGLMASR
case "bailian":
return alibabaFunASRRealtime
case "whisper":
return openAIWhisper
case "mimo":
return mimoASR
case "groq":
return groqWhisper
case "siliconflow":
return siliconflowASR
case "openrouter":
return openrouterWhisper
case "volcengine":
return volcengineDefaultResourceID
case "openai":
return openAIRealtimeWhisper
case "custom":
return openAITranscribe
default:
return openAITranscribe
}
}
/// Whether the ASR settings card should expose a custom endpoint field.
public static func showsASREndpointField(for providerId: String) -> Bool {
switch strategy(for: providerId) {
case .prompt, .openRouterJson, .bailianStreaming, .openaiRealtimeStreaming:
return true
case .zhipuHotwords, .volcengineStreaming, .localFallback:
return false
}
}
}
extension LLMProvider {
public var cloudASRStrategy: CloudASRStrategy {
CloudASRModelCatalog.strategy(for: id)
}
public var defaultCloudASRModel: String {
CloudASRModelCatalog.defaultModel(for: id)
}
/// Official hotwords / vocabulary APIs during cloud ASR (not prompt-only bias).
public var supportsPersonalDictionaryCloudASR: Bool {
switch cloudASRStrategy {
case .zhipuHotwords:
return true
case .bailianStreaming, .prompt, .openRouterJson, .volcengineStreaming,
.openaiRealtimeStreaming, .localFallback:
return false
}
}
/// Product badge: true streaming ASR path is wired for this provider.
public var supportsStreamingCloudASR: Bool {
CloudASRModelCatalog.supportsTrueStreamingASR(for: id)
}
}